2024-12-01 20:34:03,408 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-01 20:34:03,430 main DEBUG Took 0.019744 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-01 20:34:03,430 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-01 20:34:03,431 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-01 20:34:03,432 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-01 20:34:03,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,442 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-01 20:34:03,456 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,459 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,460 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,462 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,463 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,464 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,464 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,465 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,465 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,466 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,466 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,467 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,468 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,468 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-01 20:34:03,469 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,469 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-01 20:34:03,471 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-01 20:34:03,472 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-01 20:34:03,473 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-01 20:34:03,474 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-01 20:34:03,475 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-01 20:34:03,475 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-01 20:34:03,484 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-01 20:34:03,486 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-01 20:34:03,487 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-01 20:34:03,488 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-01 20:34:03,488 main DEBUG createAppenders(={Console}) 2024-12-01 20:34:03,489 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-01 20:34:03,489 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-01 20:34:03,489 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-01 20:34:03,490 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-01 20:34:03,490 main DEBUG OutputStream closed 2024-12-01 20:34:03,490 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-01 20:34:03,490 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-01 20:34:03,491 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-01 20:34:03,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-01 20:34:03,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-01 20:34:03,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-01 20:34:03,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-01 20:34:03,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-01 20:34:03,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-01 20:34:03,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-01 20:34:03,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-01 20:34:03,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-01 20:34:03,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-01 20:34:03,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-01 20:34:03,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-01 20:34:03,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-01 20:34:03,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-01 20:34:03,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-01 20:34:03,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-01 20:34:03,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-01 20:34:03,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-01 20:34:03,572 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01 20:34:03,572 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-01 20:34:03,573 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-01 20:34:03,574 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-01T20:34:03,588 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-01 20:34:03,591 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-01 20:34:03,591 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-01T20:34:03,863 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25 2024-12-01T20:34:03,880 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-01T20:34:03,894 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/cluster_5ee29cae-c15d-e677-3d2e-70d984d05d2a, deleteOnExit=true 2024-12-01T20:34:03,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-01T20:34:03,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/test.cache.data in system properties and HBase conf 2024-12-01T20:34:03,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/hadoop.tmp.dir in system properties and HBase conf 2024-12-01T20:34:03,896 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/hadoop.log.dir in system properties and HBase conf 2024-12-01T20:34:03,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-01T20:34:03,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-01T20:34:03,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-01T20:34:03,983 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-01T20:34:04,069 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-01T20:34:04,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-01T20:34:04,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-01T20:34:04,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-01T20:34:04,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T20:34:04,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-01T20:34:04,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-01T20:34:04,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-01T20:34:04,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T20:34:04,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-01T20:34:04,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/nfs.dump.dir in system properties and HBase conf 2024-12-01T20:34:04,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/java.io.tmpdir in system properties and HBase conf 2024-12-01T20:34:04,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-01T20:34:04,077 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-01T20:34:04,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-01T20:34:04,974 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-01T20:34:05,061 INFO [Time-limited test {}] log.Log(170): Logging initialized @2483ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-01T20:34:05,146 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:34:05,212 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T20:34:05,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:34:05,234 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:34:05,236 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T20:34:05,248 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:34:05,250 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@660c56f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:34:05,251 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45f63f41{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T20:34:05,448 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59bae7b3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/java.io.tmpdir/jetty-localhost-34683-hadoop-hdfs-3_4_1-tests_jar-_-any-12279413844030500307/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T20:34:05,455 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f237650{HTTP/1.1, (http/1.1)}{localhost:34683} 2024-12-01T20:34:05,456 INFO [Time-limited test {}] server.Server(415): Started @2879ms 2024-12-01T20:34:05,862 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-01T20:34:05,869 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-01T20:34:05,871 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-01T20:34:05,871 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-01T20:34:05,872 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-01T20:34:05,873 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d304465{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/hadoop.log.dir/,AVAILABLE} 2024-12-01T20:34:05,874 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c1ee62c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-01T20:34:05,997 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@258822a0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/java.io.tmpdir/jetty-localhost-33915-hadoop-hdfs-3_4_1-tests_jar-_-any-9157594207505940445/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T20:34:05,998 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@514e86a2{HTTP/1.1, (http/1.1)}{localhost:33915} 2024-12-01T20:34:05,998 INFO [Time-limited test {}] server.Server(415): Started @3422ms 2024-12-01T20:34:06,057 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-01T20:34:06,555 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/cluster_5ee29cae-c15d-e677-3d2e-70d984d05d2a/data/data1/current/BP-1961397349-172.17.0.2-1733085244688/current, will proceed with Du for space computation calculation, 2024-12-01T20:34:06,555 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/cluster_5ee29cae-c15d-e677-3d2e-70d984d05d2a/data/data2/current/BP-1961397349-172.17.0.2-1733085244688/current, will proceed with Du for space computation calculation, 2024-12-01T20:34:06,606 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-01T20:34:06,669 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb871c5945219ab0a with lease ID 0x3e102759f20b809f: Processing first storage report for DS-e0e9be62-7b7d-491c-9ee3-a8c82a6ca7cb from datanode DatanodeRegistration(127.0.0.1:35463, datanodeUuid=fb7319d7-5440-41d8-9628-ddc680e7b2f0, infoPort=34849, infoSecurePort=0, ipcPort=38683, storageInfo=lv=-57;cid=testClusterID;nsid=1297091348;c=1733085244688) 2024-12-01T20:34:06,670 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb871c5945219ab0a with lease ID 0x3e102759f20b809f: from storage DS-e0e9be62-7b7d-491c-9ee3-a8c82a6ca7cb node DatanodeRegistration(127.0.0.1:35463, datanodeUuid=fb7319d7-5440-41d8-9628-ddc680e7b2f0, infoPort=34849, infoSecurePort=0, ipcPort=38683, storageInfo=lv=-57;cid=testClusterID;nsid=1297091348;c=1733085244688), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-01T20:34:06,671 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb871c5945219ab0a with lease ID 0x3e102759f20b809f: Processing first storage report for DS-1a591ec7-524d-4a91-820b-e9221e45b88c from datanode DatanodeRegistration(127.0.0.1:35463, datanodeUuid=fb7319d7-5440-41d8-9628-ddc680e7b2f0, infoPort=34849, infoSecurePort=0, ipcPort=38683, storageInfo=lv=-57;cid=testClusterID;nsid=1297091348;c=1733085244688) 2024-12-01T20:34:06,671 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb871c5945219ab0a with lease ID 0x3e102759f20b809f: from storage DS-1a591ec7-524d-4a91-820b-e9221e45b88c node DatanodeRegistration(127.0.0.1:35463, datanodeUuid=fb7319d7-5440-41d8-9628-ddc680e7b2f0, infoPort=34849, infoSecurePort=0, ipcPort=38683, storageInfo=lv=-57;cid=testClusterID;nsid=1297091348;c=1733085244688), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-01T20:34:06,769 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25 2024-12-01T20:34:06,874 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/cluster_5ee29cae-c15d-e677-3d2e-70d984d05d2a/zookeeper_0, clientPort=56459, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/cluster_5ee29cae-c15d-e677-3d2e-70d984d05d2a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/cluster_5ee29cae-c15d-e677-3d2e-70d984d05d2a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-01T20:34:06,884 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56459 2024-12-01T20:34:06,900 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:34:06,904 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:34:07,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741825_1001 (size=7) 2024-12-01T20:34:07,566 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d with version=8 2024-12-01T20:34:07,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/hbase-staging 2024-12-01T20:34:07,679 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-01T20:34:07,942 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2140e1fa03ff:0 server-side Connection retries=9 2024-12-01T20:34:07,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T20:34:07,954 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T20:34:07,958 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T20:34:07,958 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T20:34:07,959 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T20:34:08,158 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-01T20:34:08,289 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-01T20:34:08,305 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-01T20:34:08,312 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T20:34:08,354 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15015 (auto-detected) 2024-12-01T20:34:08,358 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-01T20:34:08,395 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33377 2024-12-01T20:34:08,429 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33377 connecting to ZooKeeper ensemble=127.0.0.1:56459 2024-12-01T20:34:08,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:333770x0, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T20:34:08,540 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33377-0x1016fbc20130000 connected 2024-12-01T20:34:08,642 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:34:08,646 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:34:08,685 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T20:34:08,699 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d, hbase.cluster.distributed=false 2024-12-01T20:34:08,761 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T20:34:08,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33377 2024-12-01T20:34:08,773 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33377 2024-12-01T20:34:08,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33377 2024-12-01T20:34:08,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33377 2024-12-01T20:34:08,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33377 2024-12-01T20:34:09,002 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2140e1fa03ff:0 server-side Connection retries=9 2024-12-01T20:34:09,005 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T20:34:09,007 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-01T20:34:09,009 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-01T20:34:09,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-01T20:34:09,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-01T20:34:09,013 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-01T20:34:09,020 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-01T20:34:09,031 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38225 2024-12-01T20:34:09,037 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38225 connecting to ZooKeeper ensemble=127.0.0.1:56459 2024-12-01T20:34:09,039 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:34:09,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:34:09,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382250x0, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-01T20:34:09,082 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:382250x0, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T20:34:09,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38225-0x1016fbc20130001 connected 2024-12-01T20:34:09,088 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-01T20:34:09,103 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-01T20:34:09,106 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-01T20:34:09,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-01T20:34:09,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38225 2024-12-01T20:34:09,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38225 2024-12-01T20:34:09,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38225 2024-12-01T20:34:09,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38225 2024-12-01T20:34:09,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38225 2024-12-01T20:34:09,251 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2140e1fa03ff:33377 2024-12-01T20:34:09,252 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2140e1fa03ff,33377,1733085247739 2024-12-01T20:34:09,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T20:34:09,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T20:34:09,280 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2140e1fa03ff,33377,1733085247739 2024-12-01T20:34:09,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-01T20:34:09,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:09,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:09,329 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-01T20:34:09,333 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2140e1fa03ff,33377,1733085247739 from backup master directory 2024-12-01T20:34:09,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T20:34:09,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2140e1fa03ff,33377,1733085247739 2024-12-01T20:34:09,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-01T20:34:09,339 WARN [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T20:34:09,339 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2140e1fa03ff,33377,1733085247739 2024-12-01T20:34:09,343 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-01T20:34:09,357 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-01T20:34:09,450 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/hbase.id] with ID: 70b673ee-3e01-45b2-b3d7-d93ce7bf75e0 2024-12-01T20:34:09,451 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/.tmp/hbase.id 2024-12-01T20:34:09,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741826_1002 (size=42) 2024-12-01T20:34:09,488 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/.tmp/hbase.id]:[hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/hbase.id] 2024-12-01T20:34:09,582 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-01T20:34:09,592 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-01T20:34:09,631 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 30ms. 2024-12-01T20:34:09,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:09,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:09,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741827_1003 (size=196) 2024-12-01T20:34:10,157 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:34:10,162 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-01T20:34:10,185 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T20:34:10,192 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T20:34:10,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741828_1004 (size=1189) 2024-12-01T20:34:10,292 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store 2024-12-01T20:34:10,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741829_1005 (size=34) 2024-12-01T20:34:10,736 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-01T20:34:10,740 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:34:10,741 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T20:34:10,742 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:34:10,742 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:34:10,744 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T20:34:10,744 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:34:10,744 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:34:10,746 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733085250741Disabling compacts and flushes for region at 1733085250741Disabling writes for close at 1733085250744 (+3 ms)Writing region close event to WAL at 1733085250744Closed at 1733085250744 2024-12-01T20:34:10,762 WARN [master/2140e1fa03ff:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/.initializing 2024-12-01T20:34:10,762 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/WALs/2140e1fa03ff,33377,1733085247739 2024-12-01T20:34:10,778 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T20:34:10,808 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2140e1fa03ff%2C33377%2C1733085247739, suffix=, logDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/WALs/2140e1fa03ff,33377,1733085247739, archiveDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/oldWALs, maxLogs=10 2024-12-01T20:34:10,842 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/WALs/2140e1fa03ff,33377,1733085247739/2140e1fa03ff%2C33377%2C1733085247739.1733085250814, exclude list is [], retry=0 2024-12-01T20:34:10,876 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35463,DS-e0e9be62-7b7d-491c-9ee3-a8c82a6ca7cb,DISK] 2024-12-01T20:34:10,880 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-01T20:34:10,939 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/WALs/2140e1fa03ff,33377,1733085247739/2140e1fa03ff%2C33377%2C1733085247739.1733085250814 2024-12-01T20:34:10,940 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34849:34849)] 2024-12-01T20:34:10,941 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:34:10,942 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:34:10,946 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:10,947 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:10,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,053 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-01T20:34:11,058 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:11,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:34:11,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-01T20:34:11,068 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:11,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:34:11,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,076 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-01T20:34:11,076 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:11,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:34:11,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,081 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-01T20:34:11,081 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:11,082 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:34:11,083 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,087 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,088 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,095 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,096 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,100 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T20:34:11,105 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-01T20:34:11,122 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:34:11,123 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70595774, jitterRate=0.05195900797843933}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T20:34:11,134 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733085250962Initializing all the Stores at 1733085250965 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733085250966 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085250967 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085250967Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085250967Cleaning up temporary data from old regions at 1733085251096 (+129 ms)Region opened successfully at 1733085251133 (+37 ms) 2024-12-01T20:34:11,136 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-01T20:34:11,185 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@84b03f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2140e1fa03ff/172.17.0.2:0 2024-12-01T20:34:11,238 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-01T20:34:11,252 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-01T20:34:11,253 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-01T20:34:11,257 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-01T20:34:11,258 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-01T20:34:11,266 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-01T20:34:11,266 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-01T20:34:11,307 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-01T20:34:11,319 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-01T20:34:11,332 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-01T20:34:11,336 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-01T20:34:11,339 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-01T20:34:11,344 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-01T20:34:11,349 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-01T20:34:11,360 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-01T20:34:11,364 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-01T20:34:11,366 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-01T20:34:11,367 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-01T20:34:11,395 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-01T20:34:11,400 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-01T20:34:11,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T20:34:11,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:11,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-01T20:34:11,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:11,411 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2140e1fa03ff,33377,1733085247739, sessionid=0x1016fbc20130000, setting cluster-up flag (Was=false) 2024-12-01T20:34:11,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:11,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:11,459 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-01T20:34:11,464 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2140e1fa03ff,33377,1733085247739 2024-12-01T20:34:11,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:11,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:34:11,491 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-01T20:34:11,494 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2140e1fa03ff,33377,1733085247739 2024-12-01T20:34:11,506 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-01T20:34:11,531 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(746): ClusterId : 70b673ee-3e01-45b2-b3d7-d93ce7bf75e0 2024-12-01T20:34:11,537 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-01T20:34:11,546 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-01T20:34:11,546 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-01T20:34:11,550 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-01T20:34:11,551 DEBUG [RS:0;2140e1fa03ff:38225 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625214d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2140e1fa03ff/172.17.0.2:0 2024-12-01T20:34:11,612 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2140e1fa03ff:38225 2024-12-01T20:34:11,621 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-01T20:34:11,621 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-01T20:34:11,621 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-01T20:34:11,631 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(2659): reportForDuty to master=2140e1fa03ff,33377,1733085247739 with port=38225, startcode=1733085248936 2024-12-01T20:34:11,665 DEBUG [RS:0;2140e1fa03ff:38225 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-01T20:34:11,678 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-01T20:34:11,713 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-01T20:34:11,742 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-01T20:34:11,752 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2140e1fa03ff,33377,1733085247739 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-01T20:34:11,766 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59587, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-01T20:34:11,774 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33377 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-01T20:34:11,776 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2140e1fa03ff:0, corePoolSize=5, maxPoolSize=5 2024-12-01T20:34:11,776 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2140e1fa03ff:0, corePoolSize=5, maxPoolSize=5 2024-12-01T20:34:11,777 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2140e1fa03ff:0, corePoolSize=5, maxPoolSize=5 2024-12-01T20:34:11,777 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2140e1fa03ff:0, corePoolSize=5, maxPoolSize=5 2024-12-01T20:34:11,777 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2140e1fa03ff:0, corePoolSize=10, maxPoolSize=10 2024-12-01T20:34:11,778 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:11,778 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2140e1fa03ff:0, corePoolSize=2, maxPoolSize=2 2024-12-01T20:34:11,778 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:11,815 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-01T20:34:11,815 WARN [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-12-01T20:34:11,832 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T20:34:11,834 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-01T20:34:11,841 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733085281841 2024-12-01T20:34:11,843 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-01T20:34:11,845 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-01T20:34:11,850 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:11,851 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-01T20:34:11,852 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-01T20:34:11,850 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-01T20:34:11,853 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-01T20:34:11,854 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-01T20:34:11,877 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:11,917 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-01T20:34:11,918 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-01T20:34:11,918 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-01T20:34:11,933 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-01T20:34:11,933 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-01T20:34:11,940 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2140e1fa03ff:0:becomeActiveMaster-HFileCleaner.large.0-1733085251935,5,FailOnTimeoutGroup] 2024-12-01T20:34:11,943 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2140e1fa03ff:0:becomeActiveMaster-HFileCleaner.small.0-1733085251940,5,FailOnTimeoutGroup] 2024-12-01T20:34:11,944 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:11,944 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-01T20:34:11,946 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:11,947 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:11,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741831_1007 (size=1321) 2024-12-01T20:34:11,958 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-01T20:34:11,958 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:34:12,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741832_1008 (size=32) 2024-12-01T20:34:12,019 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:34:12,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T20:34:12,030 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T20:34:12,030 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:12,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:34:12,034 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T20:34:12,040 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T20:34:12,040 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:12,041 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:34:12,042 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T20:34:12,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T20:34:12,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:12,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:34:12,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T20:34:12,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T20:34:12,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:34:12,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:34:12,093 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T20:34:12,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740 2024-12-01T20:34:12,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740 2024-12-01T20:34:12,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T20:34:12,100 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T20:34:12,102 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T20:34:12,108 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T20:34:12,121 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:34:12,124 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74580732, jitterRate=0.1113395094871521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T20:34:12,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733085252019Initializing all the Stores at 1733085252024 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733085252024Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733085252024Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085252024Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733085252024Cleaning up temporary data from old regions at 1733085252101 (+77 ms)Region opened successfully at 1733085252129 (+28 ms) 2024-12-01T20:34:12,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T20:34:12,129 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T20:34:12,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T20:34:12,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T20:34:12,130 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T20:34:12,132 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T20:34:12,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733085252129Disabling compacts and flushes for region at 1733085252129Disabling writes for close at 1733085252130 (+1 ms)Writing region close event to WAL at 1733085252131 (+1 ms)Closed at 1733085252132 (+1 ms) 2024-12-01T20:34:12,146 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T20:34:12,147 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-01T20:34:12,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-01T20:34:12,177 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T20:34:12,185 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-01T20:34:12,336 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:13,337 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:14,338 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:15,338 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:16,339 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:17,340 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:18,273 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-01T20:34:18,273 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-01T20:34:18,341 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:19,341 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:20,342 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:21,343 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:22,344 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:23,345 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:24,345 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:25,346 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:26,347 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:27,348 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:28,348 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:29,349 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:30,350 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:31,350 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:32,351 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:33,352 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:34,352 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:35,353 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:36,354 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:36,749 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T20:34:37,354 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:38,355 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:39,356 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:40,357 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:41,357 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:41,833 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(2659): reportForDuty to master=2140e1fa03ff,33377,1733085247739 with port=38225, startcode=1733085248936 2024-12-01T20:34:41,836 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33377 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2140e1fa03ff,38225,1733085248936 2024-12-01T20:34:41,839 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33377 {}] master.ServerManager(517): Registering regionserver=2140e1fa03ff,38225,1733085248936 2024-12-01T20:34:41,850 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:34:41,850 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40247 2024-12-01T20:34:41,850 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-01T20:34:41,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T20:34:41,856 DEBUG [RS:0;2140e1fa03ff:38225 {}] zookeeper.ZKUtil(111): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2140e1fa03ff,38225,1733085248936 2024-12-01T20:34:41,856 WARN [RS:0;2140e1fa03ff:38225 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-01T20:34:41,856 INFO [RS:0;2140e1fa03ff:38225 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T20:34:41,856 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/WALs/2140e1fa03ff,38225,1733085248936 2024-12-01T20:34:41,858 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2140e1fa03ff,38225,1733085248936] 2024-12-01T20:34:41,885 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-01T20:34:41,898 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-01T20:34:41,904 INFO [RS:0;2140e1fa03ff:38225 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-01T20:34:41,904 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,905 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-01T20:34:41,911 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-01T20:34:41,913 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,914 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,914 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,914 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,914 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,915 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,915 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2140e1fa03ff:0, corePoolSize=2, maxPoolSize=2 2024-12-01T20:34:41,915 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,915 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,915 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,915 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,916 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,916 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2140e1fa03ff:0, corePoolSize=1, maxPoolSize=1 2024-12-01T20:34:41,916 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2140e1fa03ff:0, corePoolSize=3, maxPoolSize=3 2024-12-01T20:34:41,916 DEBUG [RS:0;2140e1fa03ff:38225 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2140e1fa03ff:0, corePoolSize=3, maxPoolSize=3 2024-12-01T20:34:41,917 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,917 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,918 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,918 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,918 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,918 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=2140e1fa03ff,38225,1733085248936-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T20:34:41,944 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-01T20:34:41,946 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=2140e1fa03ff,38225,1733085248936-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,947 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,947 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.Replication(171): 2140e1fa03ff,38225,1733085248936 started 2024-12-01T20:34:41,967 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:34:41,968 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1482): Serving as 2140e1fa03ff,38225,1733085248936, RpcServer on 2140e1fa03ff/172.17.0.2:38225, sessionid=0x1016fbc20130001 2024-12-01T20:34:41,969 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-01T20:34:41,969 DEBUG [RS:0;2140e1fa03ff:38225 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2140e1fa03ff,38225,1733085248936 2024-12-01T20:34:41,969 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2140e1fa03ff,38225,1733085248936' 2024-12-01T20:34:41,969 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-01T20:34:41,971 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-01T20:34:41,971 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-01T20:34:41,972 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-01T20:34:41,972 DEBUG [RS:0;2140e1fa03ff:38225 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2140e1fa03ff,38225,1733085248936 2024-12-01T20:34:41,972 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2140e1fa03ff,38225,1733085248936' 2024-12-01T20:34:41,972 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-01T20:34:41,973 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-01T20:34:41,973 DEBUG [RS:0;2140e1fa03ff:38225 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-01T20:34:41,973 INFO [RS:0;2140e1fa03ff:38225 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-01T20:34:41,973 INFO [RS:0;2140e1fa03ff:38225 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-01T20:34:42,358 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:43,359 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:44,360 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:45,360 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:46,361 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:47,362 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:48,362 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:49,363 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:50,364 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:51,364 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:52,365 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:53,366 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:54,367 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:55,367 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:56,368 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:57,369 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:58,370 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:34:59,370 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:00,371 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:01,371 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:02,372 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:03,373 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:04,373 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:05,374 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:06,375 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:06,749 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T20:35:07,375 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:08,376 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:09,377 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:10,377 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:11,378 WARN [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-01T20:35:11,980 INFO [RS:0;2140e1fa03ff:38225 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-01T20:35:11,985 INFO [RS:0;2140e1fa03ff:38225 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2140e1fa03ff%2C38225%2C1733085248936, suffix=, logDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/WALs/2140e1fa03ff,38225,1733085248936, archiveDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/oldWALs, maxLogs=32 2024-12-01T20:35:12,010 DEBUG [RS:0;2140e1fa03ff:38225 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/WALs/2140e1fa03ff,38225,1733085248936/2140e1fa03ff%2C38225%2C1733085248936.1733085311989, exclude list is [], retry=0 2024-12-01T20:35:12,016 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35463,DS-e0e9be62-7b7d-491c-9ee3-a8c82a6ca7cb,DISK] 2024-12-01T20:35:12,020 INFO [RS:0;2140e1fa03ff:38225 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/WALs/2140e1fa03ff,38225,1733085248936/2140e1fa03ff%2C38225%2C1733085248936.1733085311989 2024-12-01T20:35:12,021 DEBUG [RS:0;2140e1fa03ff:38225 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34849:34849)] 2024-12-01T20:35:12,131 DEBUG [2140e1fa03ff:33377 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-01T20:35:12,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:12,155 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2140e1fa03ff,38225,1733085248936, state=OPENING 2024-12-01T20:35:12,162 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-01T20:35:12,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:35:12,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:35:12,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T20:35:12,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T20:35:12,166 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-01T20:35:12,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:12,366 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-01T20:35:12,375 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43469, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-01T20:35:12,395 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-01T20:35:12,395 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-01T20:35:12,396 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-01T20:35:12,401 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2140e1fa03ff%2C38225%2C1733085248936.meta, suffix=.meta, logDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/WALs/2140e1fa03ff,38225,1733085248936, archiveDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/oldWALs, maxLogs=32 2024-12-01T20:35:12,422 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/WALs/2140e1fa03ff,38225,1733085248936/2140e1fa03ff%2C38225%2C1733085248936.meta.1733085312403.meta, exclude list is [], retry=0 2024-12-01T20:35:12,426 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35463,DS-e0e9be62-7b7d-491c-9ee3-a8c82a6ca7cb,DISK] 2024-12-01T20:35:12,430 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/WALs/2140e1fa03ff,38225,1733085248936/2140e1fa03ff%2C38225%2C1733085248936.meta.1733085312403.meta 2024-12-01T20:35:12,431 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34849:34849)] 2024-12-01T20:35:12,431 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:12,433 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-01T20:35:12,436 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-01T20:35:12,442 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-01T20:35:12,448 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-01T20:35:12,448 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:12,449 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-01T20:35:12,449 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-01T20:35:12,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-01T20:35:12,458 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-01T20:35:12,458 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:12,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:35:12,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-01T20:35:12,466 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-01T20:35:12,466 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:12,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:35:12,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-01T20:35:12,472 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-01T20:35:12,472 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:12,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:35:12,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-01T20:35:12,475 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-01T20:35:12,475 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:12,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-01T20:35:12,476 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-01T20:35:12,478 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740 2024-12-01T20:35:12,482 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740 2024-12-01T20:35:12,485 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-01T20:35:12,485 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-01T20:35:12,486 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-01T20:35:12,490 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-01T20:35:12,492 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68298933, jitterRate=0.017733410000801086}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-01T20:35:12,493 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-01T20:35:12,495 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733085312450Writing region info on filesystem at 1733085312450Initializing all the Stores at 1733085312452 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733085312452Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733085312453 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085312453Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733085312453Cleaning up temporary data from old regions at 1733085312485 (+32 ms)Running coprocessor post-open hooks at 1733085312493 (+8 ms)Region opened successfully at 1733085312495 (+2 ms) 2024-12-01T20:35:12,504 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733085312345 2024-12-01T20:35:12,521 DEBUG [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-01T20:35:12,522 INFO [RS_OPEN_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-01T20:35:12,523 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:12,526 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2140e1fa03ff,38225,1733085248936, state=OPEN 2024-12-01T20:35:12,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T20:35:12,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-01T20:35:12,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T20:35:12,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-01T20:35:12,536 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:12,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-01T20:35:12,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2140e1fa03ff,38225,1733085248936 in 368 msec 2024-12-01T20:35:12,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-01T20:35:12,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.388 sec 2024-12-01T20:35:12,563 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-01T20:35:12,563 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-01T20:35:12,591 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:35:12,593 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2140e1fa03ff,38225,1733085248936, seqNum=-1] 2024-12-01T20:35:12,628 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:35:12,632 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33539, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:35:12,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 1.1 sec 2024-12-01T20:35:12,664 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733085312664, completionTime=-1 2024-12-01T20:35:12,668 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-01T20:35:12,668 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-01T20:35:12,703 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-01T20:35:12,703 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733085372703 2024-12-01T20:35:12,703 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733085432703 2024-12-01T20:35:12,703 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 34 msec 2024-12-01T20:35:12,715 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2140e1fa03ff,33377,1733085247739-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-01T20:35:12,716 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2140e1fa03ff,33377,1733085247739-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:35:12,716 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2140e1fa03ff,33377,1733085247739-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:35:12,718 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2140e1fa03ff:33377, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:35:12,718 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-01T20:35:12,719 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-01T20:35:12,764 DEBUG [master/2140e1fa03ff:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-01T20:35:12,806 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 63.466sec 2024-12-01T20:35:12,808 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-01T20:35:12,812 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-01T20:35:12,814 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-01T20:35:12,814 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-01T20:35:12,815 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-01T20:35:12,815 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2140e1fa03ff,33377,1733085247739-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-01T20:35:12,816 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2140e1fa03ff,33377,1733085247739-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-01T20:35:12,833 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-01T20:35:12,835 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-01T20:35:12,835 INFO [master/2140e1fa03ff:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2140e1fa03ff,33377,1733085247739-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-01T20:35:12,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@400440bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T20:35:12,898 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-01T20:35:12,898 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-01T20:35:12,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2140e1fa03ff,33377,-1 for getting cluster id 2024-12-01T20:35:12,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:35:12,915 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '70b673ee-3e01-45b2-b3d7-d93ce7bf75e0' 2024-12-01T20:35:12,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:35:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "70b673ee-3e01-45b2-b3d7-d93ce7bf75e0" 2024-12-01T20:35:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@397ce3e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T20:35:12,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2140e1fa03ff,33377,-1] 2024-12-01T20:35:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:35:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:35:12,929 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:35:12,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675d8ae3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T20:35:12,934 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:35:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2140e1fa03ff,38225,1733085248936, seqNum=-1] 2024-12-01T20:35:12,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:35:12,948 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:35:12,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2140e1fa03ff,33377,1733085247739 2024-12-01T20:35:12,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de36109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T20:35:12,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2140e1fa03ff,33377,-1 for getting cluster id 2024-12-01T20:35:12,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-01T20:35:12,986 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '70b673ee-3e01-45b2-b3d7-d93ce7bf75e0' 2024-12-01T20:35:12,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-01T20:35:12,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "70b673ee-3e01-45b2-b3d7-d93ce7bf75e0" 2024-12-01T20:35:12,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee2f7f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T20:35:12,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2140e1fa03ff,33377,-1] 2024-12-01T20:35:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-01T20:35:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:35:12,991 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-01T20:35:12,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29a9f507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-01T20:35:13,013 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=5926 2024-12-01T20:35:13,020 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-01T20:35:13,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2140e1fa03ff,33377,1733085247739 2024-12-01T20:35:13,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@76c5f21 2024-12-01T20:35:13,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-01T20:35:13,034 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-01T20:35:13,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-01T20:35:13,055 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:13,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-01T20:35:13,059 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:13,062 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T20:35:13,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741835_1011 (size=428) 2024-12-01T20:35:13,082 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d16d6d9a6d0117a579e46840c314c902, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:13,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741836_1012 (size=79) 2024-12-01T20:35:13,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:13,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing d16d6d9a6d0117a579e46840c314c902, disabling compactions & flushes 2024-12-01T20:35:13,508 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:35:13,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:35:13,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. after waiting 0 ms 2024-12-01T20:35:13,508 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:35:13,508 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:35:13,509 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for d16d6d9a6d0117a579e46840c314c902: Waiting for close lock at 1733085313508Disabling compacts and flushes for region at 1733085313508Disabling writes for close at 1733085313508Writing region close event to WAL at 1733085313508Closed at 1733085313508 2024-12-01T20:35:13,512 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:13,518 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733085313512"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085313512"}]},"ts":"1733085313512"} 2024-12-01T20:35:13,525 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:13,527 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:13,531 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085313527"}]},"ts":"1733085313527"} 2024-12-01T20:35:13,537 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-01T20:35:13,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=d16d6d9a6d0117a579e46840c314c902, ASSIGN}] 2024-12-01T20:35:13,543 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=d16d6d9a6d0117a579e46840c314c902, ASSIGN 2024-12-01T20:35:13,546 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=d16d6d9a6d0117a579e46840c314c902, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:13,698 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d16d6d9a6d0117a579e46840c314c902, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:13,703 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=d16d6d9a6d0117a579e46840c314c902, ASSIGN because future has completed 2024-12-01T20:35:13,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d16d6d9a6d0117a579e46840c314c902, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:13,868 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:35:13,868 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d16d6d9a6d0117a579e46840c314c902, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:13,869 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,869 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:13,869 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,870 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,873 INFO [StoreOpener-d16d6d9a6d0117a579e46840c314c902-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,876 INFO [StoreOpener-d16d6d9a6d0117a579e46840c314c902-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d16d6d9a6d0117a579e46840c314c902 columnFamilyName testFamily 2024-12-01T20:35:13,876 DEBUG [StoreOpener-d16d6d9a6d0117a579e46840c314c902-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:13,877 INFO [StoreOpener-d16d6d9a6d0117a579e46840c314c902-1 {}] regionserver.HStore(327): Store=d16d6d9a6d0117a579e46840c314c902/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:13,877 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,879 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,880 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,881 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,881 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,884 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,890 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:13,890 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d16d6d9a6d0117a579e46840c314c902; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66036649, jitterRate=-0.015977248549461365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:13,891 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:35:13,892 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d16d6d9a6d0117a579e46840c314c902: Running coprocessor pre-open hook at 1733085313870Writing region info on filesystem at 1733085313870Initializing all the Stores at 1733085313872 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085313872Cleaning up temporary data from old regions at 1733085313881 (+9 ms)Running coprocessor post-open hooks at 1733085313891 (+10 ms)Region opened successfully at 1733085313892 (+1 ms) 2024-12-01T20:35:13,894 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902., pid=6, masterSystemTime=1733085313860 2024-12-01T20:35:13,899 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:35:13,899 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:35:13,900 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d16d6d9a6d0117a579e46840c314c902, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:13,905 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d16d6d9a6d0117a579e46840c314c902, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:13,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-01T20:35:13,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d16d6d9a6d0117a579e46840c314c902, server=2140e1fa03ff,38225,1733085248936 in 204 msec 2024-12-01T20:35:13,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-01T20:35:13,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=d16d6d9a6d0117a579e46840c314c902, ASSIGN in 374 msec 2024-12-01T20:35:13,920 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:13,920 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085313920"}]},"ts":"1733085313920"} 2024-12-01T20:35:13,924 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-01T20:35:13,926 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:13,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 881 msec 2024-12-01T20:35:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-01T20:35:14,091 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-01T20:35:14,091 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-01T20:35:14,093 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:14,100 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-01T20:35:14,101 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:14,101 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-01T20:35:14,105 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-01T20:35:14,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2140e1fa03ff,38225,1733085248936, seqNum=-1] 2024-12-01T20:35:14,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-01T20:35:14,111 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-01T20:35:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:14,139 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:14,144 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-01T20:35:14,150 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-01T20:35:14,156 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-01T20:35:14,158 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-01T20:35:14,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6027998381031292920 2024-12-01T20:35:14,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54910 deadline: 1733085317158, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-01T20:35:14,170 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-01T20:35:14,173 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-01T20:35:14,174 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:14,176 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:14,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6027998381031292918 2024-12-01T20:35:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54910 deadline: 1733085317176, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-01T20:35:14,182 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:14,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6027998381031292917 2024-12-01T20:35:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54910 deadline: 1733085317181, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-01T20:35:14,188 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:14,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6027998381031292916 2024-12-01T20:35:14,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54910 deadline: 1733085317187, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-01T20:35:14,195 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:14,212 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-524378997_22 at /127.0.0.1:50208 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=332 (was 332), ProcessCount=11 (was 11), AvailableMemoryMB=5912 (was 5926) 2024-12-01T20:35:14,224 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=332, ProcessCount=11, AvailableMemoryMB=5912 2024-12-01T20:35:14,228 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:14,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-01T20:35:14,235 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:14,235 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:14,235 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-01T20:35:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-01T20:35:14,239 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:14,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741837_1013 (size=414) 2024-12-01T20:35:14,684 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bfeefd88a6a17552e95dbeefb9f016a3, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:14,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741838_1014 (size=65) 2024-12-01T20:35:14,697 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:14,697 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing bfeefd88a6a17552e95dbeefb9f016a3, disabling compactions & flushes 2024-12-01T20:35:14,697 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:35:14,697 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:35:14,697 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. after waiting 0 ms 2024-12-01T20:35:14,697 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:35:14,697 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:35:14,697 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for bfeefd88a6a17552e95dbeefb9f016a3: Waiting for close lock at 1733085314697Disabling compacts and flushes for region at 1733085314697Disabling writes for close at 1733085314697Writing region close event to WAL at 1733085314697Closed at 1733085314697 2024-12-01T20:35:14,700 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:14,700 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733085314700"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085314700"}]},"ts":"1733085314700"} 2024-12-01T20:35:14,704 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:14,707 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:14,707 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085314707"}]},"ts":"1733085314707"} 2024-12-01T20:35:14,710 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-01T20:35:14,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=bfeefd88a6a17552e95dbeefb9f016a3, ASSIGN}] 2024-12-01T20:35:14,714 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=bfeefd88a6a17552e95dbeefb9f016a3, ASSIGN 2024-12-01T20:35:14,716 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=bfeefd88a6a17552e95dbeefb9f016a3, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:14,867 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bfeefd88a6a17552e95dbeefb9f016a3, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:14,873 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=bfeefd88a6a17552e95dbeefb9f016a3, ASSIGN because future has completed 2024-12-01T20:35:14,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfeefd88a6a17552e95dbeefb9f016a3, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:15,039 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:35:15,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => bfeefd88a6a17552e95dbeefb9f016a3, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:15,039 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,040 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:15,040 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,040 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,042 INFO [StoreOpener-bfeefd88a6a17552e95dbeefb9f016a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,044 INFO [StoreOpener-bfeefd88a6a17552e95dbeefb9f016a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfeefd88a6a17552e95dbeefb9f016a3 columnFamilyName testFamily 2024-12-01T20:35:15,045 DEBUG [StoreOpener-bfeefd88a6a17552e95dbeefb9f016a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:15,046 INFO [StoreOpener-bfeefd88a6a17552e95dbeefb9f016a3-1 {}] regionserver.HStore(327): Store=bfeefd88a6a17552e95dbeefb9f016a3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:15,046 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,047 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,048 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,048 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,048 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,053 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,057 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:15,057 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened bfeefd88a6a17552e95dbeefb9f016a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72051833, jitterRate=0.07365597784519196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:15,058 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:35:15,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for bfeefd88a6a17552e95dbeefb9f016a3: Running coprocessor pre-open hook at 1733085315040Writing region info on filesystem at 1733085315040Initializing all the Stores at 1733085315042 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085315042Cleaning up temporary data from old regions at 1733085315048 (+6 ms)Running coprocessor post-open hooks at 1733085315058 (+10 ms)Region opened successfully at 1733085315059 (+1 ms) 2024-12-01T20:35:15,060 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3., pid=9, masterSystemTime=1733085315032 2024-12-01T20:35:15,066 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:35:15,066 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:35:15,067 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bfeefd88a6a17552e95dbeefb9f016a3, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:15,071 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfeefd88a6a17552e95dbeefb9f016a3, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:15,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-01T20:35:15,082 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure bfeefd88a6a17552e95dbeefb9f016a3, server=2140e1fa03ff,38225,1733085248936 in 197 msec 2024-12-01T20:35:15,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-01T20:35:15,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=bfeefd88a6a17552e95dbeefb9f016a3, ASSIGN in 371 msec 2024-12-01T20:35:15,091 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:15,091 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085315091"}]},"ts":"1733085315091"} 2024-12-01T20:35:15,095 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-01T20:35:15,098 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:15,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 870 msec 2024-12-01T20:35:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-01T20:35:15,250 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-01T20:35:15,250 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-01T20:35:15,250 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:15,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-01T20:35:15,255 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:15,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-01T20:35:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:15,273 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:15,274 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-01T20:35:15,277 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-01T20:35:15,279 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-01T20:35:17,991 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T20:35:18,021 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-01T20:35:18,023 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-01T20:35:18,024 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-01T20:35:18,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-01T20:35:18,272 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:18,274 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T20:35:18,274 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-01T20:35:18,274 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-01T20:35:18,274 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:18,275 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-01T20:35:18,275 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:21,289 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2140e1fa03ff:38225 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-12-01T20:35:21,290 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-12-01T20:35:21,290 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 from cache 2024-12-01T20:35:21,304 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=337 (was 332) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5874 (was 5912) 2024-12-01T20:35:21,316 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=337, ProcessCount=11, AvailableMemoryMB=5874 2024-12-01T20:35:21,320 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:21,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-01T20:35:21,325 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:21,325 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:21,326 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-01T20:35:21,327 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-01T20:35:21,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741839_1015 (size=412) 2024-12-01T20:35:21,344 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 75654364910ee969aae253638987b290, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:21,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741840_1016 (size=63) 2024-12-01T20:35:21,757 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:21,757 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 75654364910ee969aae253638987b290, disabling compactions & flushes 2024-12-01T20:35:21,757 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:35:21,757 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:35:21,757 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. after waiting 0 ms 2024-12-01T20:35:21,757 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:35:21,757 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:35:21,757 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 75654364910ee969aae253638987b290: Waiting for close lock at 1733085321757Disabling compacts and flushes for region at 1733085321757Disabling writes for close at 1733085321757Writing region close event to WAL at 1733085321757Closed at 1733085321757 2024-12-01T20:35:21,760 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:21,761 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733085321760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085321760"}]},"ts":"1733085321760"} 2024-12-01T20:35:21,764 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:21,766 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:21,766 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085321766"}]},"ts":"1733085321766"} 2024-12-01T20:35:21,769 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-01T20:35:21,770 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=75654364910ee969aae253638987b290, ASSIGN}] 2024-12-01T20:35:21,771 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=75654364910ee969aae253638987b290, ASSIGN 2024-12-01T20:35:21,773 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=75654364910ee969aae253638987b290, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:21,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-12-01T20:35:21,925 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=75654364910ee969aae253638987b290, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:21,929 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=75654364910ee969aae253638987b290, ASSIGN because future has completed 2024-12-01T20:35:21,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75654364910ee969aae253638987b290, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:22,090 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:35:22,090 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 75654364910ee969aae253638987b290, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:22,090 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 75654364910ee969aae253638987b290 2024-12-01T20:35:22,090 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:22,091 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 75654364910ee969aae253638987b290 2024-12-01T20:35:22,091 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 75654364910ee969aae253638987b290 2024-12-01T20:35:22,093 INFO [StoreOpener-75654364910ee969aae253638987b290-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 75654364910ee969aae253638987b290 2024-12-01T20:35:22,095 INFO [StoreOpener-75654364910ee969aae253638987b290-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 75654364910ee969aae253638987b290 columnFamilyName testFamily 2024-12-01T20:35:22,096 DEBUG [StoreOpener-75654364910ee969aae253638987b290-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:22,096 INFO [StoreOpener-75654364910ee969aae253638987b290-1 {}] regionserver.HStore(327): Store=75654364910ee969aae253638987b290/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:22,097 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 75654364910ee969aae253638987b290 2024-12-01T20:35:22,098 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290 2024-12-01T20:35:22,099 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290 2024-12-01T20:35:22,100 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 75654364910ee969aae253638987b290 2024-12-01T20:35:22,100 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 75654364910ee969aae253638987b290 2024-12-01T20:35:22,103 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 75654364910ee969aae253638987b290 2024-12-01T20:35:22,107 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:22,108 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 75654364910ee969aae253638987b290; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72056205, jitterRate=0.07372112572193146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:22,108 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 75654364910ee969aae253638987b290 2024-12-01T20:35:22,110 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 75654364910ee969aae253638987b290: Running coprocessor pre-open hook at 1733085322091Writing region info on filesystem at 1733085322091Initializing all the Stores at 1733085322093 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085322093Cleaning up temporary data from old regions at 1733085322100 (+7 ms)Running coprocessor post-open hooks at 1733085322108 (+8 ms)Region opened successfully at 1733085322110 (+2 ms) 2024-12-01T20:35:22,111 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290., pid=12, masterSystemTime=1733085322083 2024-12-01T20:35:22,115 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:35:22,116 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:35:22,117 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=75654364910ee969aae253638987b290, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:22,122 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 75654364910ee969aae253638987b290, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:22,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-01T20:35:22,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 75654364910ee969aae253638987b290, server=2140e1fa03ff,38225,1733085248936 in 194 msec 2024-12-01T20:35:22,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-01T20:35:22,135 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=75654364910ee969aae253638987b290, ASSIGN in 360 msec 2024-12-01T20:35:22,136 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:22,137 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085322137"}]},"ts":"1733085322137"} 2024-12-01T20:35:22,141 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-01T20:35:22,142 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:22,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 822 msec 2024-12-01T20:35:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-01T20:35:22,341 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-01T20:35:22,341 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-01T20:35:22,341 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:22,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-01T20:35:22,347 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:22,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-01T20:35:22,352 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:22,369 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:22,369 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-01T20:35:22,373 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-01T20:35:22,381 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-01T20:35:22,383 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-01T20:35:22,384 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-01T20:35:24,026 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-01T20:35:27,593 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T20:35:28,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-01T20:35:28,272 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:28,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38225 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292907' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-01T20:35:28,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38225 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:54910 deadline: 1733085334396, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292907' does not exist 2024-12-01T20:35:28,400 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 2140e1fa03ff,38225,1733085248936 for closing scanner id = -6027998381031292907 for 75654364910ee969aae253638987b290 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292907' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T20:35:28,402 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 337) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5870 (was 5874) 2024-12-01T20:35:28,413 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=5870 2024-12-01T20:35:28,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-01T20:35:28,420 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:28,420 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:28,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-01T20:35:28,423 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-01T20:35:28,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741841_1017 (size=423) 2024-12-01T20:35:28,839 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a958fa583d96defe82ac625d03bd54ac, NAME => 'testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:28,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741842_1018 (size=74) 2024-12-01T20:35:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-12-01T20:35:29,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:29,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing a958fa583d96defe82ac625d03bd54ac, disabling compactions & flushes 2024-12-01T20:35:29,257 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:35:29,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:35:29,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. after waiting 0 ms 2024-12-01T20:35:29,257 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:35:29,258 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:35:29,258 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for a958fa583d96defe82ac625d03bd54ac: Waiting for close lock at 1733085329257Disabling compacts and flushes for region at 1733085329257Disabling writes for close at 1733085329257Writing region close event to WAL at 1733085329257Closed at 1733085329257 2024-12-01T20:35:29,260 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:29,260 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733085329260"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085329260"}]},"ts":"1733085329260"} 2024-12-01T20:35:29,264 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:29,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:29,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085329266"}]},"ts":"1733085329266"} 2024-12-01T20:35:29,269 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-01T20:35:29,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a958fa583d96defe82ac625d03bd54ac, ASSIGN}] 2024-12-01T20:35:29,272 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a958fa583d96defe82ac625d03bd54ac, ASSIGN 2024-12-01T20:35:29,274 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a958fa583d96defe82ac625d03bd54ac, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:29,425 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a958fa583d96defe82ac625d03bd54ac, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:29,429 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a958fa583d96defe82ac625d03bd54ac, ASSIGN because future has completed 2024-12-01T20:35:29,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a958fa583d96defe82ac625d03bd54ac, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-01T20:35:29,590 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:35:29,590 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => a958fa583d96defe82ac625d03bd54ac, NAME => 'testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:29,590 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,590 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:29,591 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,591 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,593 INFO [StoreOpener-a958fa583d96defe82ac625d03bd54ac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,597 INFO [StoreOpener-a958fa583d96defe82ac625d03bd54ac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a958fa583d96defe82ac625d03bd54ac columnFamilyName testFamily 2024-12-01T20:35:29,597 DEBUG [StoreOpener-a958fa583d96defe82ac625d03bd54ac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:29,598 INFO [StoreOpener-a958fa583d96defe82ac625d03bd54ac-1 {}] regionserver.HStore(327): Store=a958fa583d96defe82ac625d03bd54ac/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:29,598 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,599 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,600 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,601 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,601 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,604 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,607 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:29,608 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened a958fa583d96defe82ac625d03bd54ac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58862321, jitterRate=-0.12288306653499603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:29,609 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:35:29,609 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for a958fa583d96defe82ac625d03bd54ac: Running coprocessor pre-open hook at 1733085329591Writing region info on filesystem at 1733085329591Initializing all the Stores at 1733085329592 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085329593 (+1 ms)Cleaning up temporary data from old regions at 1733085329601 (+8 ms)Running coprocessor post-open hooks at 1733085329609 (+8 ms)Region opened successfully at 1733085329609 2024-12-01T20:35:29,611 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac., pid=15, masterSystemTime=1733085329583 2024-12-01T20:35:29,615 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:35:29,615 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:35:29,616 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a958fa583d96defe82ac625d03bd54ac, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:29,619 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a958fa583d96defe82ac625d03bd54ac, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:29,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-01T20:35:29,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure a958fa583d96defe82ac625d03bd54ac, server=2140e1fa03ff,38225,1733085248936 in 191 msec 2024-12-01T20:35:29,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-01T20:35:29,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a958fa583d96defe82ac625d03bd54ac, ASSIGN in 355 msec 2024-12-01T20:35:29,630 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:29,630 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085329630"}]},"ts":"1733085329630"} 2024-12-01T20:35:29,633 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-01T20:35:29,635 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:29,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2190 sec 2024-12-01T20:35:30,027 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-01T20:35:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-01T20:35:31,449 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-01T20:35:31,449 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-01T20:35:31,450 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:31,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-01T20:35:31,454 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:31,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-01T20:35:31,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:31,468 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:31,468 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-01T20:35:31,471 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-01T20:35:31,473 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-01T20:35:31,474 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-01T20:35:31,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6027998381031292903 2024-12-01T20:35:31,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54910 deadline: 1733085334474, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-01T20:35:31,478 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-01T20:35:31,479 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-01T20:35:31,480 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:31,481 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:31,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6027998381031292901 2024-12-01T20:35:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54910 deadline: 1733085334480, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-01T20:35:31,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:31,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6027998381031292900 2024-12-01T20:35:31,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54910 deadline: 1733085334483, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-01T20:35:31,487 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:31,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6027998381031292899 2024-12-01T20:35:31,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:54910 deadline: 1733085334487, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-01T20:35:31,490 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-01T20:35:31,499 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-524378997_22 at /127.0.0.1:32966 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 363), ProcessCount=11 (was 11), AvailableMemoryMB=5867 (was 5870) 2024-12-01T20:35:31,507 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=5867 2024-12-01T20:35:31,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-01T20:35:31,512 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:31,512 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:31,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-01T20:35:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-01T20:35:31,514 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:31,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741843_1019 (size=418) 2024-12-01T20:35:31,925 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df47fdce6ccecf49660d439921792837, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:31,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741844_1020 (size=69) 2024-12-01T20:35:32,334 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:32,334 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing df47fdce6ccecf49660d439921792837, disabling compactions & flushes 2024-12-01T20:35:32,334 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:35:32,334 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:35:32,334 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. after waiting 0 ms 2024-12-01T20:35:32,334 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:35:32,334 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:35:32,334 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for df47fdce6ccecf49660d439921792837: Waiting for close lock at 1733085332334Disabling compacts and flushes for region at 1733085332334Disabling writes for close at 1733085332334Writing region close event to WAL at 1733085332334Closed at 1733085332334 2024-12-01T20:35:32,336 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:32,337 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733085332337"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085332337"}]},"ts":"1733085332337"} 2024-12-01T20:35:32,340 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:32,341 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:32,342 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085332341"}]},"ts":"1733085332341"} 2024-12-01T20:35:32,344 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-01T20:35:32,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=df47fdce6ccecf49660d439921792837, ASSIGN}] 2024-12-01T20:35:32,346 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=df47fdce6ccecf49660d439921792837, ASSIGN 2024-12-01T20:35:32,347 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=df47fdce6ccecf49660d439921792837, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:32,498 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=df47fdce6ccecf49660d439921792837, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:32,501 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=df47fdce6ccecf49660d439921792837, ASSIGN because future has completed 2024-12-01T20:35:32,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure df47fdce6ccecf49660d439921792837, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:32,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-01T20:35:32,661 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:35:32,661 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => df47fdce6ccecf49660d439921792837, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:32,662 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,662 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:32,662 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,662 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,664 INFO [StoreOpener-df47fdce6ccecf49660d439921792837-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,666 INFO [StoreOpener-df47fdce6ccecf49660d439921792837-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df47fdce6ccecf49660d439921792837 columnFamilyName testFamily 2024-12-01T20:35:32,667 DEBUG [StoreOpener-df47fdce6ccecf49660d439921792837-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:32,667 INFO [StoreOpener-df47fdce6ccecf49660d439921792837-1 {}] regionserver.HStore(327): Store=df47fdce6ccecf49660d439921792837/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:32,667 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,668 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,669 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,670 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,670 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,672 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,675 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:32,676 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened df47fdce6ccecf49660d439921792837; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67242784, jitterRate=0.001995563507080078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:32,676 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df47fdce6ccecf49660d439921792837 2024-12-01T20:35:32,676 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for df47fdce6ccecf49660d439921792837: Running coprocessor pre-open hook at 1733085332662Writing region info on filesystem at 1733085332662Initializing all the Stores at 1733085332664 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085332664Cleaning up temporary data from old regions at 1733085332670 (+6 ms)Running coprocessor post-open hooks at 1733085332676 (+6 ms)Region opened successfully at 1733085332676 2024-12-01T20:35:32,678 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., pid=18, masterSystemTime=1733085332654 2024-12-01T20:35:32,681 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:35:32,681 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:35:32,681 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=df47fdce6ccecf49660d439921792837, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:32,684 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure df47fdce6ccecf49660d439921792837, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:32,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-01T20:35:32,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure df47fdce6ccecf49660d439921792837, server=2140e1fa03ff,38225,1733085248936 in 184 msec 2024-12-01T20:35:32,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-01T20:35:32,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=df47fdce6ccecf49660d439921792837, ASSIGN in 344 msec 2024-12-01T20:35:32,692 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:32,692 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085332692"}]},"ts":"1733085332692"} 2024-12-01T20:35:32,694 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-01T20:35:32,695 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:32,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1860 sec 2024-12-01T20:35:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-01T20:35:34,550 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-01T20:35:34,550 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-01T20:35:34,550 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:34,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-01T20:35:34,555 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:34,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-01T20:35:34,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:34,569 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:34,569 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-01T20:35:34,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-01T20:35:34,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-01T20:35:35,092 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T20:35:35,582 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2140e1fa03ff:38225 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-01T20:35:35,582 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-01T20:35:35,582 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 from cache 2024-12-01T20:35:36,029 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-01T20:35:36,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-12-01T20:35:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:36,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-01T20:35:36,750 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T20:35:37,581 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2140e1fa03ff:38225 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=979ms, rpcTimeout=970ms 2024-12-01T20:35:37,581 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=979ms, rpcTimeout=970ms 2024-12-01T20:35:37,581 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 from cache 2024-12-01T20:35:37,592 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=307 (was 363), ProcessCount=11 (was 11), AvailableMemoryMB=5858 (was 5867) 2024-12-01T20:35:37,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=307, ProcessCount=11, AvailableMemoryMB=5858 2024-12-01T20:35:37,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:37,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-01T20:35:37,608 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:37,608 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:37,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-01T20:35:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-01T20:35:37,610 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:37,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741845_1021 (size=416) 2024-12-01T20:35:37,625 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cdd8dab68cf23077f389e888fc959cc8, NAME => 'testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:37,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741846_1022 (size=67) 2024-12-01T20:35:37,636 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:37,636 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing cdd8dab68cf23077f389e888fc959cc8, disabling compactions & flushes 2024-12-01T20:35:37,636 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:35:37,636 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:35:37,636 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. after waiting 0 ms 2024-12-01T20:35:37,636 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:35:37,636 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:35:37,637 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for cdd8dab68cf23077f389e888fc959cc8: Waiting for close lock at 1733085337636Disabling compacts and flushes for region at 1733085337636Disabling writes for close at 1733085337636Writing region close event to WAL at 1733085337636Closed at 1733085337636 2024-12-01T20:35:37,639 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:37,639 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733085337639"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085337639"}]},"ts":"1733085337639"} 2024-12-01T20:35:37,642 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:37,644 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:37,644 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085337644"}]},"ts":"1733085337644"} 2024-12-01T20:35:37,646 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-01T20:35:37,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=cdd8dab68cf23077f389e888fc959cc8, ASSIGN}] 2024-12-01T20:35:37,648 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=cdd8dab68cf23077f389e888fc959cc8, ASSIGN 2024-12-01T20:35:37,650 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=cdd8dab68cf23077f389e888fc959cc8, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:37,800 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=cdd8dab68cf23077f389e888fc959cc8, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:37,804 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=cdd8dab68cf23077f389e888fc959cc8, ASSIGN because future has completed 2024-12-01T20:35:37,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdd8dab68cf23077f389e888fc959cc8, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:37,964 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:35:37,964 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => cdd8dab68cf23077f389e888fc959cc8, NAME => 'testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:37,965 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,965 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:37,965 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,965 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,969 INFO [StoreOpener-cdd8dab68cf23077f389e888fc959cc8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,971 INFO [StoreOpener-cdd8dab68cf23077f389e888fc959cc8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cdd8dab68cf23077f389e888fc959cc8 columnFamilyName testFamily 2024-12-01T20:35:37,971 DEBUG [StoreOpener-cdd8dab68cf23077f389e888fc959cc8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:37,972 INFO [StoreOpener-cdd8dab68cf23077f389e888fc959cc8-1 {}] regionserver.HStore(327): Store=cdd8dab68cf23077f389e888fc959cc8/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:37,972 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,973 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,974 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,974 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,974 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,977 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,982 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:37,982 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened cdd8dab68cf23077f389e888fc959cc8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63198347, jitterRate=-0.058271244168281555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:37,982 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:35:37,983 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for cdd8dab68cf23077f389e888fc959cc8: Running coprocessor pre-open hook at 1733085337965Writing region info on filesystem at 1733085337965Initializing all the Stores at 1733085337968 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085337968Cleaning up temporary data from old regions at 1733085337974 (+6 ms)Running coprocessor post-open hooks at 1733085337982 (+8 ms)Region opened successfully at 1733085337983 (+1 ms) 2024-12-01T20:35:37,984 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8., pid=21, masterSystemTime=1733085337957 2024-12-01T20:35:37,987 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:35:37,987 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:35:37,988 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=cdd8dab68cf23077f389e888fc959cc8, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:37,991 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure cdd8dab68cf23077f389e888fc959cc8, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:37,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-01T20:35:37,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure cdd8dab68cf23077f389e888fc959cc8, server=2140e1fa03ff,38225,1733085248936 in 188 msec 2024-12-01T20:35:37,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-01T20:35:37,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=cdd8dab68cf23077f389e888fc959cc8, ASSIGN in 350 msec 2024-12-01T20:35:38,000 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:38,000 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085338000"}]},"ts":"1733085338000"} 2024-12-01T20:35:38,003 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-01T20:35:38,004 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:38,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 400 msec 2024-12-01T20:35:38,104 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-12-01T20:35:38,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-01T20:35:38,272 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:38,273 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-01T20:35:38,273 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:38,273 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-01T20:35:38,273 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:38,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-01T20:35:38,630 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-01T20:35:38,630 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-01T20:35:38,630 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:38,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-01T20:35:38,636 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:38,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-01T20:35:38,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:38,652 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:38,652 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-01T20:35:38,655 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-01T20:35:38,658 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-01T20:35:38,659 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-01T20:35:38,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-01T20:35:41,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38225 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292888' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-01T20:35:41,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38225 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:54910 deadline: 1733085342664, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292888' does not exist 2024-12-01T20:35:41,665 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 2140e1fa03ff,38225,1733085248936 for closing scanner id = -6027998381031292888 for cdd8dab68cf23077f389e888fc959cc8 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292888' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T20:35:41,672 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-524378997_22 at /127.0.0.1:37940 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 307), ProcessCount=11 (was 11), AvailableMemoryMB=5847 (was 5858) 2024-12-01T20:35:41,680 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=5847 2024-12-01T20:35:41,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-01T20:35:41,686 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:41,686 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:41,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-01T20:35:41,687 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-01T20:35:41,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741847_1023 (size=411) 2024-12-01T20:35:41,699 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 677d820e3f811f6fffe1f870ec494183, NAME => 'testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:41,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741848_1024 (size=62) 2024-12-01T20:35:41,707 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:41,707 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 677d820e3f811f6fffe1f870ec494183, disabling compactions & flushes 2024-12-01T20:35:41,707 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:35:41,707 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:35:41,707 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. after waiting 0 ms 2024-12-01T20:35:41,707 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:35:41,707 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:35:41,707 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 677d820e3f811f6fffe1f870ec494183: Waiting for close lock at 1733085341707Disabling compacts and flushes for region at 1733085341707Disabling writes for close at 1733085341707Writing region close event to WAL at 1733085341707Closed at 1733085341707 2024-12-01T20:35:41,710 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:41,710 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733085341710"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085341710"}]},"ts":"1733085341710"} 2024-12-01T20:35:41,713 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:41,715 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:41,715 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085341715"}]},"ts":"1733085341715"} 2024-12-01T20:35:41,718 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-01T20:35:41,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=677d820e3f811f6fffe1f870ec494183, ASSIGN}] 2024-12-01T20:35:41,720 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=677d820e3f811f6fffe1f870ec494183, ASSIGN 2024-12-01T20:35:41,723 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=677d820e3f811f6fffe1f870ec494183, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:41,874 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=677d820e3f811f6fffe1f870ec494183, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:41,878 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=677d820e3f811f6fffe1f870ec494183, ASSIGN because future has completed 2024-12-01T20:35:41,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 677d820e3f811f6fffe1f870ec494183, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:42,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-01T20:35:42,037 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:35:42,037 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 677d820e3f811f6fffe1f870ec494183, NAME => 'testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:42,038 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,038 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:42,038 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,038 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,040 INFO [StoreOpener-677d820e3f811f6fffe1f870ec494183-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,044 INFO [StoreOpener-677d820e3f811f6fffe1f870ec494183-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 677d820e3f811f6fffe1f870ec494183 columnFamilyName testFamily 2024-12-01T20:35:42,044 DEBUG [StoreOpener-677d820e3f811f6fffe1f870ec494183-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:42,045 INFO [StoreOpener-677d820e3f811f6fffe1f870ec494183-1 {}] regionserver.HStore(327): Store=677d820e3f811f6fffe1f870ec494183/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:42,045 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,046 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,047 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,053 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,053 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,060 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,063 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:42,064 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 677d820e3f811f6fffe1f870ec494183; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71350452, jitterRate=0.06320458650588989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:42,064 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:35:42,065 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 677d820e3f811f6fffe1f870ec494183: Running coprocessor pre-open hook at 1733085342038Writing region info on filesystem at 1733085342038Initializing all the Stores at 1733085342040 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085342040Cleaning up temporary data from old regions at 1733085342053 (+13 ms)Running coprocessor post-open hooks at 1733085342064 (+11 ms)Region opened successfully at 1733085342065 (+1 ms) 2024-12-01T20:35:42,067 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183., pid=24, masterSystemTime=1733085342031 2024-12-01T20:35:42,070 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:35:42,070 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:35:42,071 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=677d820e3f811f6fffe1f870ec494183, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:42,074 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 677d820e3f811f6fffe1f870ec494183, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:42,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-01T20:35:42,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 677d820e3f811f6fffe1f870ec494183, server=2140e1fa03ff,38225,1733085248936 in 197 msec 2024-12-01T20:35:42,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-01T20:35:42,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=677d820e3f811f6fffe1f870ec494183, ASSIGN in 360 msec 2024-12-01T20:35:42,082 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:42,082 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085342082"}]},"ts":"1733085342082"} 2024-12-01T20:35:42,085 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-01T20:35:42,086 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:42,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 404 msec 2024-12-01T20:35:42,160 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-01T20:35:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-01T20:35:42,700 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-01T20:35:42,700 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-01T20:35:42,700 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:42,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-01T20:35:42,706 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:42,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-01T20:35:42,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:42,723 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:42,723 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-01T20:35:42,728 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-01T20:35:42,733 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-01T20:35:42,735 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-01T20:35:42,736 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-01T20:35:43,467 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T20:35:45,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38225 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292884' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-01T20:35:45,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38225 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:54910 deadline: 1733085346744, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292884' does not exist 2024-12-01T20:35:45,745 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 2140e1fa03ff,38225,1733085248936 for closing scanner id = -6027998381031292884 for 677d820e3f811f6fffe1f870ec494183 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292884' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T20:35:45,752 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=306 (was 306), ProcessCount=11 (was 11), AvailableMemoryMB=5837 (was 5847) 2024-12-01T20:35:45,760 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=306, ProcessCount=11, AvailableMemoryMB=5837 2024-12-01T20:35:45,763 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:45,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-01T20:35:45,766 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:45,766 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:45,766 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-01T20:35:45,768 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-01T20:35:45,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741849_1025 (size=419) 2024-12-01T20:35:46,189 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9b46fcfcaaeb971d81cc45dd2da6f676, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:46,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741850_1026 (size=70) 2024-12-01T20:35:46,201 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:46,201 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9b46fcfcaaeb971d81cc45dd2da6f676, disabling compactions & flushes 2024-12-01T20:35:46,201 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:35:46,201 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:35:46,202 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. after waiting 0 ms 2024-12-01T20:35:46,202 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:35:46,202 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:35:46,202 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9b46fcfcaaeb971d81cc45dd2da6f676: Waiting for close lock at 1733085346201Disabling compacts and flushes for region at 1733085346201Disabling writes for close at 1733085346202 (+1 ms)Writing region close event to WAL at 1733085346202Closed at 1733085346202 2024-12-01T20:35:46,203 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:46,204 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733085346204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085346204"}]},"ts":"1733085346204"} 2024-12-01T20:35:46,208 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:46,210 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:46,210 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085346210"}]},"ts":"1733085346210"} 2024-12-01T20:35:46,213 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-01T20:35:46,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9b46fcfcaaeb971d81cc45dd2da6f676, ASSIGN}] 2024-12-01T20:35:46,215 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9b46fcfcaaeb971d81cc45dd2da6f676, ASSIGN 2024-12-01T20:35:46,216 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9b46fcfcaaeb971d81cc45dd2da6f676, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:46,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-12-01T20:35:46,367 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=9b46fcfcaaeb971d81cc45dd2da6f676, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:46,371 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9b46fcfcaaeb971d81cc45dd2da6f676, ASSIGN because future has completed 2024-12-01T20:35:46,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b46fcfcaaeb971d81cc45dd2da6f676, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:46,531 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:35:46,531 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b46fcfcaaeb971d81cc45dd2da6f676, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:46,531 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:46,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,534 INFO [StoreOpener-9b46fcfcaaeb971d81cc45dd2da6f676-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,536 INFO [StoreOpener-9b46fcfcaaeb971d81cc45dd2da6f676-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b46fcfcaaeb971d81cc45dd2da6f676 columnFamilyName testFamily 2024-12-01T20:35:46,536 DEBUG [StoreOpener-9b46fcfcaaeb971d81cc45dd2da6f676-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:46,539 INFO [StoreOpener-9b46fcfcaaeb971d81cc45dd2da6f676-1 {}] regionserver.HStore(327): Store=9b46fcfcaaeb971d81cc45dd2da6f676/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:46,539 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,544 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,545 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,545 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,546 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,551 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,554 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:46,555 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 9b46fcfcaaeb971d81cc45dd2da6f676; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68504720, jitterRate=0.020799875259399414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:46,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:35:46,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 9b46fcfcaaeb971d81cc45dd2da6f676: Running coprocessor pre-open hook at 1733085346532Writing region info on filesystem at 1733085346532Initializing all the Stores at 1733085346533 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085346533Cleaning up temporary data from old regions at 1733085346546 (+13 ms)Running coprocessor post-open hooks at 1733085346555 (+9 ms)Region opened successfully at 1733085346556 (+1 ms) 2024-12-01T20:35:46,557 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676., pid=27, masterSystemTime=1733085346524 2024-12-01T20:35:46,560 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:35:46,560 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:35:46,561 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=9b46fcfcaaeb971d81cc45dd2da6f676, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:46,564 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b46fcfcaaeb971d81cc45dd2da6f676, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:46,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-01T20:35:46,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b46fcfcaaeb971d81cc45dd2da6f676, server=2140e1fa03ff,38225,1733085248936 in 194 msec 2024-12-01T20:35:46,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-01T20:35:46,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9b46fcfcaaeb971d81cc45dd2da6f676, ASSIGN in 355 msec 2024-12-01T20:35:46,573 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:46,573 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085346573"}]},"ts":"1733085346573"} 2024-12-01T20:35:46,576 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-01T20:35:46,577 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:46,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 814 msec 2024-12-01T20:35:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-01T20:35:46,780 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-01T20:35:46,780 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-01T20:35:46,780 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:46,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-01T20:35:46,785 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:46,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-01T20:35:46,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:46,799 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:46,799 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-01T20:35:46,803 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-01T20:35:46,804 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-01T20:35:48,031 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-01T20:35:48,032 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-01T20:35:48,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-01T20:35:48,272 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:48,273 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-01T20:35:48,273 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:52,034 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T20:35:52,811 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2140e1fa03ff:38225 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-01T20:35:52,811 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-01T20:35:52,811 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 from cache 2024-12-01T20:35:52,823 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 306) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5830 (was 5837) 2024-12-01T20:35:52,832 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=5830 2024-12-01T20:35:52,835 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:52,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-01T20:35:52,838 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:52,839 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:52,839 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-01T20:35:52,840 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-01T20:35:52,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741851_1027 (size=407) 2024-12-01T20:35:52,851 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e5a1c461af1d6f06b4e5f7f5eb85e89, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:35:52,857 INFO [master/2140e1fa03ff:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-01T20:35:52,857 INFO [master/2140e1fa03ff:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-01T20:35:52,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741852_1028 (size=58) 2024-12-01T20:35:53,265 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:53,265 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 1e5a1c461af1d6f06b4e5f7f5eb85e89, disabling compactions & flushes 2024-12-01T20:35:53,265 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:35:53,265 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:35:53,265 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. after waiting 0 ms 2024-12-01T20:35:53,265 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:35:53,265 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:35:53,266 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e5a1c461af1d6f06b4e5f7f5eb85e89: Waiting for close lock at 1733085353265Disabling compacts and flushes for region at 1733085353265Disabling writes for close at 1733085353265Writing region close event to WAL at 1733085353265Closed at 1733085353265 2024-12-01T20:35:53,267 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:35:53,268 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733085353267"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085353267"}]},"ts":"1733085353267"} 2024-12-01T20:35:53,272 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:35:53,273 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:35:53,273 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085353273"}]},"ts":"1733085353273"} 2024-12-01T20:35:53,276 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-01T20:35:53,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1e5a1c461af1d6f06b4e5f7f5eb85e89, ASSIGN}] 2024-12-01T20:35:53,279 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1e5a1c461af1d6f06b4e5f7f5eb85e89, ASSIGN 2024-12-01T20:35:53,280 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1e5a1c461af1d6f06b4e5f7f5eb85e89, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:35:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-12-01T20:35:53,431 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1e5a1c461af1d6f06b4e5f7f5eb85e89, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:53,435 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1e5a1c461af1d6f06b4e5f7f5eb85e89, ASSIGN because future has completed 2024-12-01T20:35:53,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e5a1c461af1d6f06b4e5f7f5eb85e89, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:35:53,595 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:35:53,595 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e5a1c461af1d6f06b4e5f7f5eb85e89, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:35:53,595 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,596 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:35:53,596 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,596 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,598 INFO [StoreOpener-1e5a1c461af1d6f06b4e5f7f5eb85e89-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,600 INFO [StoreOpener-1e5a1c461af1d6f06b4e5f7f5eb85e89-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e5a1c461af1d6f06b4e5f7f5eb85e89 columnFamilyName testFamily 2024-12-01T20:35:53,600 DEBUG [StoreOpener-1e5a1c461af1d6f06b4e5f7f5eb85e89-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:53,601 INFO [StoreOpener-1e5a1c461af1d6f06b4e5f7f5eb85e89-1 {}] regionserver.HStore(327): Store=1e5a1c461af1d6f06b4e5f7f5eb85e89/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:35:53,601 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,602 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,603 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,603 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,603 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,606 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,609 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:35:53,611 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 1e5a1c461af1d6f06b4e5f7f5eb85e89; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72358417, jitterRate=0.07822443544864655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:35:53,611 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:35:53,612 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 1e5a1c461af1d6f06b4e5f7f5eb85e89: Running coprocessor pre-open hook at 1733085353596Writing region info on filesystem at 1733085353596Initializing all the Stores at 1733085353597 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085353597Cleaning up temporary data from old regions at 1733085353603 (+6 ms)Running coprocessor post-open hooks at 1733085353611 (+8 ms)Region opened successfully at 1733085353612 (+1 ms) 2024-12-01T20:35:53,614 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89., pid=30, masterSystemTime=1733085353588 2024-12-01T20:35:53,617 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:35:53,617 INFO [RS_OPEN_PRIORITY_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:35:53,618 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1e5a1c461af1d6f06b4e5f7f5eb85e89, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:35:53,622 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e5a1c461af1d6f06b4e5f7f5eb85e89, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:35:53,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-01T20:35:53,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e5a1c461af1d6f06b4e5f7f5eb85e89, server=2140e1fa03ff,38225,1733085248936 in 189 msec 2024-12-01T20:35:53,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-01T20:35:53,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1e5a1c461af1d6f06b4e5f7f5eb85e89, ASSIGN in 350 msec 2024-12-01T20:35:53,630 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:35:53,630 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085353630"}]},"ts":"1733085353630"} 2024-12-01T20:35:53,633 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-01T20:35:53,634 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:35:53,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 799 msec 2024-12-01T20:35:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-01T20:35:53,849 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-01T20:35:53,850 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-01T20:35:53,850 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:53,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-01T20:35:53,856 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:35:53,856 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-01T20:35:53,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:35:53,871 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:35:53,872 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-01T20:35:53,874 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-01T20:35:53,876 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-01T20:35:53,877 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-01T20:35:53,878 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-01T20:35:54,033 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-01T20:35:58,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-01T20:35:58,272 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-01T20:35:59,097 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T20:35:59,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38225 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292876' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-01T20:35:59,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38225 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:54910 deadline: 1733085365882, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292876' does not exist 2024-12-01T20:35:59,886 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 2140e1fa03ff,38225,1733085248936 for closing scanner id = -6027998381031292876 for 1e5a1c461af1d6f06b4e5f7f5eb85e89 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6027998381031292876' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-01T20:35:59,891 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=333 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=5818 (was 5830) 2024-12-01T20:35:59,899 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=333, ProcessCount=11, AvailableMemoryMB=5818 2024-12-01T20:35:59,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-01T20:35:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-01T20:35:59,904 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-01T20:35:59,904 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:35:59,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-01T20:35:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-01T20:35:59,905 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-01T20:35:59,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741853_1029 (size=423) 2024-12-01T20:36:00,316 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a82b87203de750dd68c6ac182daadd2c, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d 2024-12-01T20:36:00,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741854_1030 (size=74) 2024-12-01T20:36:00,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-12-01T20:36:00,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:36:00,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing a82b87203de750dd68c6ac182daadd2c, disabling compactions & flushes 2024-12-01T20:36:00,724 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:00,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:00,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. after waiting 0 ms 2024-12-01T20:36:00,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:00,724 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:00,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for a82b87203de750dd68c6ac182daadd2c: Waiting for close lock at 1733085360724Disabling compacts and flushes for region at 1733085360724Disabling writes for close at 1733085360724Writing region close event to WAL at 1733085360724Closed at 1733085360724 2024-12-01T20:36:00,726 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-01T20:36:00,726 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733085360726"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733085360726"}]},"ts":"1733085360726"} 2024-12-01T20:36:00,729 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-01T20:36:00,730 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-01T20:36:00,731 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085360731"}]},"ts":"1733085360731"} 2024-12-01T20:36:00,733 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-01T20:36:00,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a82b87203de750dd68c6ac182daadd2c, ASSIGN}] 2024-12-01T20:36:00,734 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a82b87203de750dd68c6ac182daadd2c, ASSIGN 2024-12-01T20:36:00,735 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a82b87203de750dd68c6ac182daadd2c, ASSIGN; state=OFFLINE, location=2140e1fa03ff,38225,1733085248936; forceNewPlan=false, retain=false 2024-12-01T20:36:00,886 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=a82b87203de750dd68c6ac182daadd2c, regionState=OPENING, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:36:00,890 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a82b87203de750dd68c6ac182daadd2c, ASSIGN because future has completed 2024-12-01T20:36:00,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure a82b87203de750dd68c6ac182daadd2c, server=2140e1fa03ff,38225,1733085248936}] 2024-12-01T20:36:00,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-01T20:36:01,048 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:01,048 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => a82b87203de750dd68c6ac182daadd2c, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c.', STARTKEY => '', ENDKEY => ''} 2024-12-01T20:36:01,048 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,049 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-01T20:36:01,049 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,049 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,051 INFO [StoreOpener-a82b87203de750dd68c6ac182daadd2c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,052 INFO [StoreOpener-a82b87203de750dd68c6ac182daadd2c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a82b87203de750dd68c6ac182daadd2c columnFamilyName testFamily 2024-12-01T20:36:01,053 DEBUG [StoreOpener-a82b87203de750dd68c6ac182daadd2c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-01T20:36:01,053 INFO [StoreOpener-a82b87203de750dd68c6ac182daadd2c-1 {}] regionserver.HStore(327): Store=a82b87203de750dd68c6ac182daadd2c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-01T20:36:01,053 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,054 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,055 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,055 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,055 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,057 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,060 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-01T20:36:01,061 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened a82b87203de750dd68c6ac182daadd2c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73386894, jitterRate=0.0935499370098114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-01T20:36:01,061 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:01,061 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for a82b87203de750dd68c6ac182daadd2c: Running coprocessor pre-open hook at 1733085361049Writing region info on filesystem at 1733085361049Initializing all the Stores at 1733085361050 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733085361050Cleaning up temporary data from old regions at 1733085361055 (+5 ms)Running coprocessor post-open hooks at 1733085361061 (+6 ms)Region opened successfully at 1733085361061 2024-12-01T20:36:01,062 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., pid=33, masterSystemTime=1733085361042 2024-12-01T20:36:01,065 DEBUG [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:01,065 INFO [RS_OPEN_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:01,066 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=a82b87203de750dd68c6ac182daadd2c, regionState=OPEN, openSeqNum=2, regionLocation=2140e1fa03ff,38225,1733085248936 2024-12-01T20:36:01,068 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure a82b87203de750dd68c6ac182daadd2c, server=2140e1fa03ff,38225,1733085248936 because future has completed 2024-12-01T20:36:01,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-01T20:36:01,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure a82b87203de750dd68c6ac182daadd2c, server=2140e1fa03ff,38225,1733085248936 in 179 msec 2024-12-01T20:36:01,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-01T20:36:01,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=a82b87203de750dd68c6ac182daadd2c, ASSIGN in 338 msec 2024-12-01T20:36:01,075 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-01T20:36:01,075 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733085361075"}]},"ts":"1733085361075"} 2024-12-01T20:36:01,077 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-01T20:36:01,078 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-01T20:36:01,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1770 sec 2024-12-01T20:36:02,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33377 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-01T20:36:02,930 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-01T20:36:02,930 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-01T20:36:02,930 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:36:02,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-01T20:36:02,935 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-01T20:36:02,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-01T20:36:02,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:36:02,948 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-01T20:36:02,948 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-01T20:36:02,951 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-01T20:36:02,952 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-01T20:36:03,962 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2140e1fa03ff:38225 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-12-01T20:36:03,962 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1010ms, rpcTimeout=1000ms 2024-12-01T20:36:03,962 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 from cache 2024-12-01T20:36:04,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-12-01T20:36:04,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2] 2024-12-01T20:36:04,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38225 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-01T20:36:05,961 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=2140e1fa03ff:38225 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=979ms, rpcTimeout=969ms 2024-12-01T20:36:05,961 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=979ms, rpcTimeout=969ms 2024-12-01T20:36:05,961 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., hostname=2140e1fa03ff,38225,1733085248936, seqNum=2 from cache 2024-12-01T20:36:05,972 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=322 (was 333), ProcessCount=11 (was 11), AvailableMemoryMB=5808 (was 5818) 2024-12-01T20:36:05,972 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:36:05,973 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:36:05,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:36:05,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:36:05,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:36:05,979 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-01T20:36:05,979 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:36:05,979 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:36:05,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:36:05,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:36:05,980 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-01T20:36:05,980 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-01T20:36:05,980 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=687218034, stopped=false 2024-12-01T20:36:05,980 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2140e1fa03ff,33377,1733085247739 2024-12-01T20:36:05,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T20:36:05,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:36:05,984 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T20:36:05,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-01T20:36:05,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:36:05,985 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-01T20:36:05,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T20:36:05,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-01T20:36:05,986 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:36:05,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:36:05,986 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2140e1fa03ff,38225,1733085248936' ***** 2024-12-01T20:36:05,987 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-01T20:36:05,987 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-01T20:36:05,987 INFO [RS:0;2140e1fa03ff:38225 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-01T20:36:05,987 INFO [RS:0;2140e1fa03ff:38225 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-01T20:36:05,988 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for 75654364910ee969aae253638987b290 2024-12-01T20:36:05,988 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for bfeefd88a6a17552e95dbeefb9f016a3 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for 9b46fcfcaaeb971d81cc45dd2da6f676 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for 677d820e3f811f6fffe1f870ec494183 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for a82b87203de750dd68c6ac182daadd2c 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for df47fdce6ccecf49660d439921792837 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for cdd8dab68cf23077f389e888fc959cc8 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for 1e5a1c461af1d6f06b4e5f7f5eb85e89 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(959): stopping server 2140e1fa03ff,38225,1733085248936 2024-12-01T20:36:05,989 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 75654364910ee969aae253638987b290, disabling compactions & flushes 2024-12-01T20:36:05,989 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T20:36:05,989 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:36:05,990 INFO [RS:0;2140e1fa03ff:38225 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2140e1fa03ff:38225. 2024-12-01T20:36:05,990 DEBUG [RS:0;2140e1fa03ff:38225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-01T20:36:05,990 DEBUG [RS:0;2140e1fa03ff:38225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:36:05,990 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-01T20:36:05,990 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-01T20:36:05,990 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-01T20:36:05,990 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-01T20:36:05,990 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-01T20:36:05,991 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-01T20:36:05,991 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-01T20:36:05,991 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-01T20:36:05,991 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-01T20:36:05,991 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-01T20:36:05,990 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:36:05,992 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1e5a1c461af1d6f06b4e5f7f5eb85e89, 677d820e3f811f6fffe1f870ec494183, 75654364910ee969aae253638987b290, 9b46fcfcaaeb971d81cc45dd2da6f676, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, bfeefd88a6a17552e95dbeefb9f016a3, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:05,992 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. after waiting 0 ms 2024-12-01T20:36:05,992 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:36:05,995 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-01T20:36:05,995 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 75654364910ee969aae253638987b290 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:06,035 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-01T20:36:06,070 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290/.tmp/testFamily/9acead30c77f4c0ba045dee4e3fcf28c is 61, key is row-0/testFamily:testQualifier/1733085322354/Put/seqid=0 2024-12-01T20:36:06,082 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/info/a933e9e64d0348de8d2c084dc36e9115 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902./info:regioninfo/1733085313900/Put/seqid=0 2024-12-01T20:36:06,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741856_1032 (size=20564) 2024-12-01T20:36:06,100 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/info/a933e9e64d0348de8d2c084dc36e9115 2024-12-01T20:36:06,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741855_1031 (size=5049) 2024-12-01T20:36:06,112 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290/.tmp/testFamily/9acead30c77f4c0ba045dee4e3fcf28c 2024-12-01T20:36:06,176 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290/.tmp/testFamily/9acead30c77f4c0ba045dee4e3fcf28c as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290/testFamily/9acead30c77f4c0ba045dee4e3fcf28c 2024-12-01T20:36:06,192 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1e5a1c461af1d6f06b4e5f7f5eb85e89, 677d820e3f811f6fffe1f870ec494183, 75654364910ee969aae253638987b290, 9b46fcfcaaeb971d81cc45dd2da6f676, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, bfeefd88a6a17552e95dbeefb9f016a3, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:06,200 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/ns/94cb01e563dd40579e592a5e73daecfa is 43, key is default/ns:d/1733085312638/Put/seqid=0 2024-12-01T20:36:06,203 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290/testFamily/9acead30c77f4c0ba045dee4e3fcf28c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:06,224 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 75654364910ee969aae253638987b290 in 224ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:06,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741857_1033 (size=5153) 2024-12-01T20:36:06,234 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/ns/94cb01e563dd40579e592a5e73daecfa 2024-12-01T20:36:06,269 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNextAsync/75654364910ee969aae253638987b290/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:06,272 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:36:06,272 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 75654364910ee969aae253638987b290: Waiting for close lock at 1733085365989Running coprocessor pre-close hooks at 1733085365989Disabling compacts and flushes for region at 1733085365989Disabling writes for close at 1733085365992 (+3 ms)Obtaining lock to block concurrent updates at 1733085365996 (+4 ms)Preparing flush snapshotting stores in 75654364910ee969aae253638987b290 at 1733085365996Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085366004 (+8 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. at 1733085366006 (+2 ms)Flushing 75654364910ee969aae253638987b290/testFamily: creating writer at 1733085366007 (+1 ms)Flushing 75654364910ee969aae253638987b290/testFamily: appending metadata at 1733085366062 (+55 ms)Flushing 75654364910ee969aae253638987b290/testFamily: closing flushed file at 1733085366066 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31d5b7f7: reopening flushed file at 1733085366165 (+99 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 75654364910ee969aae253638987b290 in 224ms, sequenceid=8, compaction requested=false at 1733085366224 (+59 ms)Writing region close event to WAL at 1733085366255 (+31 ms)Running coprocessor post-close hooks at 1733085366270 (+15 ms)Closed at 1733085366272 (+2 ms) 2024-12-01T20:36:06,275 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733085321319.75654364910ee969aae253638987b290. 2024-12-01T20:36:06,276 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bfeefd88a6a17552e95dbeefb9f016a3, disabling compactions & flushes 2024-12-01T20:36:06,276 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:36:06,276 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:36:06,276 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. after waiting 0 ms 2024-12-01T20:36:06,276 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:36:06,276 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bfeefd88a6a17552e95dbeefb9f016a3 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:06,280 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/table/948c2a8e93ea4d69ad349f679013a503 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733085313920/Put/seqid=0 2024-12-01T20:36:06,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741858_1034 (size=6699) 2024-12-01T20:36:06,307 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3/.tmp/testFamily/058284e9681744539b54cf7bf9f03e56 is 61, key is row-0/testFamily:testQualifier/1733085315260/Put/seqid=0 2024-12-01T20:36:06,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741859_1035 (size=5049) 2024-12-01T20:36:06,319 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3/.tmp/testFamily/058284e9681744539b54cf7bf9f03e56 2024-12-01T20:36:06,331 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3/.tmp/testFamily/058284e9681744539b54cf7bf9f03e56 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3/testFamily/058284e9681744539b54cf7bf9f03e56 2024-12-01T20:36:06,343 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3/testFamily/058284e9681744539b54cf7bf9f03e56, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:06,345 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bfeefd88a6a17552e95dbeefb9f016a3 in 69ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:06,369 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScanner/bfeefd88a6a17552e95dbeefb9f016a3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:06,370 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:36:06,370 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bfeefd88a6a17552e95dbeefb9f016a3: Waiting for close lock at 1733085366276Running coprocessor pre-close hooks at 1733085366276Disabling compacts and flushes for region at 1733085366276Disabling writes for close at 1733085366276Obtaining lock to block concurrent updates at 1733085366276Preparing flush snapshotting stores in bfeefd88a6a17552e95dbeefb9f016a3 at 1733085366276Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085366277 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. at 1733085366278 (+1 ms)Flushing bfeefd88a6a17552e95dbeefb9f016a3/testFamily: creating writer at 1733085366278Flushing bfeefd88a6a17552e95dbeefb9f016a3/testFamily: appending metadata at 1733085366307 (+29 ms)Flushing bfeefd88a6a17552e95dbeefb9f016a3/testFamily: closing flushed file at 1733085366307Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46833f21: reopening flushed file at 1733085366330 (+23 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bfeefd88a6a17552e95dbeefb9f016a3 in 69ms, sequenceid=8, compaction requested=false at 1733085366346 (+16 ms)Writing region close event to WAL at 1733085366352 (+6 ms)Running coprocessor post-close hooks at 1733085366370 (+18 ms)Closed at 1733085366370 2024-12-01T20:36:06,370 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733085314227.bfeefd88a6a17552e95dbeefb9f016a3. 2024-12-01T20:36:06,370 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9b46fcfcaaeb971d81cc45dd2da6f676, disabling compactions & flushes 2024-12-01T20:36:06,371 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:36:06,371 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:36:06,371 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. after waiting 0 ms 2024-12-01T20:36:06,371 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:36:06,371 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9b46fcfcaaeb971d81cc45dd2da6f676 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:06,390 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676/.tmp/testFamily/346990ddb8044772a747852f35da3c8d is 61, key is row-0/testFamily:testQualifier/1733085346789/Put/seqid=0 2024-12-01T20:36:06,392 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1e5a1c461af1d6f06b4e5f7f5eb85e89, 677d820e3f811f6fffe1f870ec494183, 9b46fcfcaaeb971d81cc45dd2da6f676, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:06,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741860_1036 (size=5049) 2024-12-01T20:36:06,484 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-01T20:36:06,551 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-01T20:36:06,592 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1e5a1c461af1d6f06b4e5f7f5eb85e89, 677d820e3f811f6fffe1f870ec494183, 9b46fcfcaaeb971d81cc45dd2da6f676, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:06,695 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/table/948c2a8e93ea4d69ad349f679013a503 2024-12-01T20:36:06,706 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/info/a933e9e64d0348de8d2c084dc36e9115 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/info/a933e9e64d0348de8d2c084dc36e9115 2024-12-01T20:36:06,718 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/info/a933e9e64d0348de8d2c084dc36e9115, entries=100, sequenceid=56, filesize=20.1 K 2024-12-01T20:36:06,719 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/ns/94cb01e563dd40579e592a5e73daecfa as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/ns/94cb01e563dd40579e592a5e73daecfa 2024-12-01T20:36:06,731 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/ns/94cb01e563dd40579e592a5e73daecfa, entries=2, sequenceid=56, filesize=5.0 K 2024-12-01T20:36:06,733 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/.tmp/table/948c2a8e93ea4d69ad349f679013a503 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/table/948c2a8e93ea4d69ad349f679013a503 2024-12-01T20:36:06,745 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/table/948c2a8e93ea4d69ad349f679013a503, entries=20, sequenceid=56, filesize=6.5 K 2024-12-01T20:36:06,747 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 756ms, sequenceid=56, compaction requested=false 2024-12-01T20:36:06,750 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-01T20:36:06,768 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-01T20:36:06,769 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T20:36:06,770 INFO [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-01T20:36:06,770 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733085365990Running coprocessor pre-close hooks at 1733085365990Disabling compacts and flushes for region at 1733085365990Disabling writes for close at 1733085365991 (+1 ms)Obtaining lock to block concurrent updates at 1733085365996 (+5 ms)Preparing flush snapshotting stores in 1588230740 at 1733085365996Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733085366004 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733085366017 (+13 ms)Flushing 1588230740/info: creating writer at 1733085366017Flushing 1588230740/info: appending metadata at 1733085366073 (+56 ms)Flushing 1588230740/info: closing flushed file at 1733085366073Flushing 1588230740/ns: creating writer at 1733085366164 (+91 ms)Flushing 1588230740/ns: appending metadata at 1733085366195 (+31 ms)Flushing 1588230740/ns: closing flushed file at 1733085366195Flushing 1588230740/table: creating writer at 1733085366247 (+52 ms)Flushing 1588230740/table: appending metadata at 1733085366279 (+32 ms)Flushing 1588230740/table: closing flushed file at 1733085366279Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4aadb49c: reopening flushed file at 1733085366705 (+426 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36027e98: reopening flushed file at 1733085366718 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39b2829c: reopening flushed file at 1733085366732 (+14 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 756ms, sequenceid=56, compaction requested=false at 1733085366747 (+15 ms)Writing region close event to WAL at 1733085366757 (+10 ms)Running coprocessor post-close hooks at 1733085366769 (+12 ms)Closed at 1733085366770 (+1 ms) 2024-12-01T20:36:06,770 DEBUG [RS_CLOSE_META-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-01T20:36:06,793 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, 677d820e3f811f6fffe1f870ec494183, 9b46fcfcaaeb971d81cc45dd2da6f676, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:06,799 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676/.tmp/testFamily/346990ddb8044772a747852f35da3c8d 2024-12-01T20:36:06,810 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676/.tmp/testFamily/346990ddb8044772a747852f35da3c8d as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676/testFamily/346990ddb8044772a747852f35da3c8d 2024-12-01T20:36:06,820 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676/testFamily/346990ddb8044772a747852f35da3c8d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:06,821 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9b46fcfcaaeb971d81cc45dd2da6f676 in 450ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:06,828 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9b46fcfcaaeb971d81cc45dd2da6f676/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:06,829 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:36:06,829 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9b46fcfcaaeb971d81cc45dd2da6f676: Waiting for close lock at 1733085366370Running coprocessor pre-close hooks at 1733085366370Disabling compacts and flushes for region at 1733085366370Disabling writes for close at 1733085366371 (+1 ms)Obtaining lock to block concurrent updates at 1733085366371Preparing flush snapshotting stores in 9b46fcfcaaeb971d81cc45dd2da6f676 at 1733085366371Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085366371Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. at 1733085366372 (+1 ms)Flushing 9b46fcfcaaeb971d81cc45dd2da6f676/testFamily: creating writer at 1733085366372Flushing 9b46fcfcaaeb971d81cc45dd2da6f676/testFamily: appending metadata at 1733085366390 (+18 ms)Flushing 9b46fcfcaaeb971d81cc45dd2da6f676/testFamily: closing flushed file at 1733085366390Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eb25f00: reopening flushed file at 1733085366809 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9b46fcfcaaeb971d81cc45dd2da6f676 in 450ms, sequenceid=8, compaction requested=false at 1733085366821 (+12 ms)Writing region close event to WAL at 1733085366822 (+1 ms)Running coprocessor post-close hooks at 1733085366828 (+6 ms)Closed at 1733085366828 2024-12-01T20:36:06,829 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733085345762.9b46fcfcaaeb971d81cc45dd2da6f676. 2024-12-01T20:36:06,829 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 677d820e3f811f6fffe1f870ec494183, disabling compactions & flushes 2024-12-01T20:36:06,829 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:36:06,829 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:36:06,829 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. after waiting 0 ms 2024-12-01T20:36:06,829 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:36:06,830 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 677d820e3f811f6fffe1f870ec494183 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:06,851 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183/.tmp/testFamily/a440a26392244077910d94334ade4e95 is 61, key is row-0/testFamily:testQualifier/1733085342710/Put/seqid=0 2024-12-01T20:36:06,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741861_1037 (size=5049) 2024-12-01T20:36:06,947 INFO [regionserver/2140e1fa03ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-01T20:36:06,948 INFO [regionserver/2140e1fa03ff:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-01T20:36:06,993 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-12-01T20:36:06,993 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1325): Online Regions={677d820e3f811f6fffe1f870ec494183=testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183., a82b87203de750dd68c6ac182daadd2c=testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., df47fdce6ccecf49660d439921792837=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., cdd8dab68cf23077f389e888fc959cc8=testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8., 1e5a1c461af1d6f06b4e5f7f5eb85e89=hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89., d16d6d9a6d0117a579e46840c314c902=testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902., a958fa583d96defe82ac625d03bd54ac=testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac.} 2024-12-01T20:36:06,993 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, 677d820e3f811f6fffe1f870ec494183, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:07,194 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, 677d820e3f811f6fffe1f870ec494183, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:07,258 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183/.tmp/testFamily/a440a26392244077910d94334ade4e95 2024-12-01T20:36:07,270 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183/.tmp/testFamily/a440a26392244077910d94334ade4e95 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183/testFamily/a440a26392244077910d94334ade4e95 2024-12-01T20:36:07,282 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183/testFamily/a440a26392244077910d94334ade4e95, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:07,283 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 677d820e3f811f6fffe1f870ec494183 in 454ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:07,289 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNext/677d820e3f811f6fffe1f870ec494183/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:07,293 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:36:07,293 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 677d820e3f811f6fffe1f870ec494183: Waiting for close lock at 1733085366829Running coprocessor pre-close hooks at 1733085366829Disabling compacts and flushes for region at 1733085366829Disabling writes for close at 1733085366829Obtaining lock to block concurrent updates at 1733085366830 (+1 ms)Preparing flush snapshotting stores in 677d820e3f811f6fffe1f870ec494183 at 1733085366830Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085366830Flushing stores of testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. at 1733085366831 (+1 ms)Flushing 677d820e3f811f6fffe1f870ec494183/testFamily: creating writer at 1733085366831Flushing 677d820e3f811f6fffe1f870ec494183/testFamily: appending metadata at 1733085366851 (+20 ms)Flushing 677d820e3f811f6fffe1f870ec494183/testFamily: closing flushed file at 1733085366851Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@de66096: reopening flushed file at 1733085367269 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 677d820e3f811f6fffe1f870ec494183 in 454ms, sequenceid=8, compaction requested=false at 1733085367283 (+14 ms)Writing region close event to WAL at 1733085367284 (+1 ms)Running coprocessor post-close hooks at 1733085367293 (+9 ms)Closed at 1733085367293 2024-12-01T20:36:07,293 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733085341682.677d820e3f811f6fffe1f870ec494183. 2024-12-01T20:36:07,293 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a82b87203de750dd68c6ac182daadd2c, disabling compactions & flushes 2024-12-01T20:36:07,293 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:07,293 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:07,294 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. after waiting 0 ms 2024-12-01T20:36:07,294 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:07,294 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a82b87203de750dd68c6ac182daadd2c 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:07,316 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c/.tmp/testFamily/f03437abd8a94e45b40044c2a4a0aa59 is 61, key is row-0/testFamily:testQualifier/1733085362939/Put/seqid=0 2024-12-01T20:36:07,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741862_1038 (size=5049) 2024-12-01T20:36:07,394 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:07,594 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a82b87203de750dd68c6ac182daadd2c, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:07,723 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c/.tmp/testFamily/f03437abd8a94e45b40044c2a4a0aa59 2024-12-01T20:36:07,732 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c/.tmp/testFamily/f03437abd8a94e45b40044c2a4a0aa59 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c/testFamily/f03437abd8a94e45b40044c2a4a0aa59 2024-12-01T20:36:07,742 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c/testFamily/f03437abd8a94e45b40044c2a4a0aa59, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:07,743 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a82b87203de750dd68c6ac182daadd2c in 449ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:07,757 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScannerAsync/a82b87203de750dd68c6ac182daadd2c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:07,758 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:07,758 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a82b87203de750dd68c6ac182daadd2c: Waiting for close lock at 1733085367293Running coprocessor pre-close hooks at 1733085367293Disabling compacts and flushes for region at 1733085367293Disabling writes for close at 1733085367294 (+1 ms)Obtaining lock to block concurrent updates at 1733085367294Preparing flush snapshotting stores in a82b87203de750dd68c6ac182daadd2c at 1733085367294Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085367294Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. at 1733085367295 (+1 ms)Flushing a82b87203de750dd68c6ac182daadd2c/testFamily: creating writer at 1733085367295Flushing a82b87203de750dd68c6ac182daadd2c/testFamily: appending metadata at 1733085367316 (+21 ms)Flushing a82b87203de750dd68c6ac182daadd2c/testFamily: closing flushed file at 1733085367316Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@678fea5b: reopening flushed file at 1733085367731 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a82b87203de750dd68c6ac182daadd2c in 449ms, sequenceid=8, compaction requested=false at 1733085367743 (+12 ms)Writing region close event to WAL at 1733085367744 (+1 ms)Running coprocessor post-close hooks at 1733085367758 (+14 ms)Closed at 1733085367758 2024-12-01T20:36:07,759 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733085359901.a82b87203de750dd68c6ac182daadd2c. 2024-12-01T20:36:07,759 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df47fdce6ccecf49660d439921792837, disabling compactions & flushes 2024-12-01T20:36:07,759 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:36:07,759 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:36:07,759 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. after waiting 0 ms 2024-12-01T20:36:07,759 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:36:07,759 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing df47fdce6ccecf49660d439921792837 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:07,786 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837/.tmp/testFamily/18512b687ed04d54b1ef29f68e0c4bcf is 61, key is row-0/testFamily:testQualifier/1733085334559/Put/seqid=0 2024-12-01T20:36:07,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741863_1039 (size=5049) 2024-12-01T20:36:07,794 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:07,995 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-12-01T20:36:07,995 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1325): Online Regions={df47fdce6ccecf49660d439921792837=testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., cdd8dab68cf23077f389e888fc959cc8=testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8., 1e5a1c461af1d6f06b4e5f7f5eb85e89=hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89., d16d6d9a6d0117a579e46840c314c902=testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902., a958fa583d96defe82ac625d03bd54ac=testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac.} 2024-12-01T20:36:07,995 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:08,195 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837/.tmp/testFamily/18512b687ed04d54b1ef29f68e0c4bcf 2024-12-01T20:36:08,195 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902, df47fdce6ccecf49660d439921792837 2024-12-01T20:36:08,207 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837/.tmp/testFamily/18512b687ed04d54b1ef29f68e0c4bcf as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837/testFamily/18512b687ed04d54b1ef29f68e0c4bcf 2024-12-01T20:36:08,216 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837/testFamily/18512b687ed04d54b1ef29f68e0c4bcf, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:08,217 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for df47fdce6ccecf49660d439921792837 in 458ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:08,222 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnOpenScanner/df47fdce6ccecf49660d439921792837/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:08,223 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:36:08,224 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df47fdce6ccecf49660d439921792837: Waiting for close lock at 1733085367759Running coprocessor pre-close hooks at 1733085367759Disabling compacts and flushes for region at 1733085367759Disabling writes for close at 1733085367759Obtaining lock to block concurrent updates at 1733085367759Preparing flush snapshotting stores in df47fdce6ccecf49660d439921792837 at 1733085367759Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085367760 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. at 1733085367761 (+1 ms)Flushing df47fdce6ccecf49660d439921792837/testFamily: creating writer at 1733085367761Flushing df47fdce6ccecf49660d439921792837/testFamily: appending metadata at 1733085367785 (+24 ms)Flushing df47fdce6ccecf49660d439921792837/testFamily: closing flushed file at 1733085367786 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60f172c4: reopening flushed file at 1733085368206 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for df47fdce6ccecf49660d439921792837 in 458ms, sequenceid=8, compaction requested=false at 1733085368217 (+11 ms)Writing region close event to WAL at 1733085368218 (+1 ms)Running coprocessor post-close hooks at 1733085368223 (+5 ms)Closed at 1733085368223 2024-12-01T20:36:08,224 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733085331509.df47fdce6ccecf49660d439921792837. 2024-12-01T20:36:08,224 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cdd8dab68cf23077f389e888fc959cc8, disabling compactions & flushes 2024-12-01T20:36:08,224 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:36:08,224 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:36:08,224 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. after waiting 0 ms 2024-12-01T20:36:08,224 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:36:08,224 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cdd8dab68cf23077f389e888fc959cc8 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:08,245 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8/.tmp/testFamily/cb31fd9d4be944a393d1be133b0b57fe is 61, key is row-0/testFamily:testQualifier/1733085338642/Put/seqid=0 2024-12-01T20:36:08,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741864_1040 (size=5049) 2024-12-01T20:36:08,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-01T20:36:08,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-01T20:36:08,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-01T20:36:08,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-01T20:36:08,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-01T20:36:08,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-01T20:36:08,272 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-01T20:36:08,395 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:36:08,596 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a958fa583d96defe82ac625d03bd54ac, cdd8dab68cf23077f389e888fc959cc8, d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:36:08,654 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8/.tmp/testFamily/cb31fd9d4be944a393d1be133b0b57fe 2024-12-01T20:36:08,664 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8/.tmp/testFamily/cb31fd9d4be944a393d1be133b0b57fe as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8/testFamily/cb31fd9d4be944a393d1be133b0b57fe 2024-12-01T20:36:08,672 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8/testFamily/cb31fd9d4be944a393d1be133b0b57fe, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:08,673 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cdd8dab68cf23077f389e888fc959cc8 in 449ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:08,678 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testNormalScanTimeoutOnNextAsync/cdd8dab68cf23077f389e888fc959cc8/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:08,679 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:36:08,679 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cdd8dab68cf23077f389e888fc959cc8: Waiting for close lock at 1733085368224Running coprocessor pre-close hooks at 1733085368224Disabling compacts and flushes for region at 1733085368224Disabling writes for close at 1733085368224Obtaining lock to block concurrent updates at 1733085368224Preparing flush snapshotting stores in cdd8dab68cf23077f389e888fc959cc8 at 1733085368224Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085368225 (+1 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. at 1733085368225Flushing cdd8dab68cf23077f389e888fc959cc8/testFamily: creating writer at 1733085368226 (+1 ms)Flushing cdd8dab68cf23077f389e888fc959cc8/testFamily: appending metadata at 1733085368244 (+18 ms)Flushing cdd8dab68cf23077f389e888fc959cc8/testFamily: closing flushed file at 1733085368244Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a02b8d5: reopening flushed file at 1733085368663 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cdd8dab68cf23077f389e888fc959cc8 in 449ms, sequenceid=8, compaction requested=false at 1733085368674 (+11 ms)Writing region close event to WAL at 1733085368674Running coprocessor post-close hooks at 1733085368679 (+5 ms)Closed at 1733085368679 2024-12-01T20:36:08,679 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733085337604.cdd8dab68cf23077f389e888fc959cc8. 2024-12-01T20:36:08,680 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1e5a1c461af1d6f06b4e5f7f5eb85e89, disabling compactions & flushes 2024-12-01T20:36:08,680 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:36:08,680 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:36:08,680 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. after waiting 0 ms 2024-12-01T20:36:08,680 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:36:08,680 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1e5a1c461af1d6f06b4e5f7f5eb85e89 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:08,698 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89/.tmp/testFamily/f5af39d21d16410580213365ae5b1e4a is 61, key is row-0/testFamily:testQualifier/1733085353861/Put/seqid=0 2024-12-01T20:36:08,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741865_1041 (size=5049) 2024-12-01T20:36:08,796 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a958fa583d96defe82ac625d03bd54ac, d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:36:08,996 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-01T20:36:08,996 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1325): Online Regions={1e5a1c461af1d6f06b4e5f7f5eb85e89=hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89., d16d6d9a6d0117a579e46840c314c902=testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902., a958fa583d96defe82ac625d03bd54ac=testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac.} 2024-12-01T20:36:08,997 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on 1e5a1c461af1d6f06b4e5f7f5eb85e89, a958fa583d96defe82ac625d03bd54ac, d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:36:09,105 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89/.tmp/testFamily/f5af39d21d16410580213365ae5b1e4a 2024-12-01T20:36:09,114 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89/.tmp/testFamily/f5af39d21d16410580213365ae5b1e4a as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89/testFamily/f5af39d21d16410580213365ae5b1e4a 2024-12-01T20:36:09,123 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89/testFamily/f5af39d21d16410580213365ae5b1e4a, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:09,124 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1e5a1c461af1d6f06b4e5f7f5eb85e89 in 444ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:09,130 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/hbase/testMetaScanTimeoutOnNext/1e5a1c461af1d6f06b4e5f7f5eb85e89/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:09,131 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:36:09,131 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1e5a1c461af1d6f06b4e5f7f5eb85e89: Waiting for close lock at 1733085368679Running coprocessor pre-close hooks at 1733085368679Disabling compacts and flushes for region at 1733085368679Disabling writes for close at 1733085368680 (+1 ms)Obtaining lock to block concurrent updates at 1733085368680Preparing flush snapshotting stores in 1e5a1c461af1d6f06b4e5f7f5eb85e89 at 1733085368680Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085368680Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. at 1733085368681 (+1 ms)Flushing 1e5a1c461af1d6f06b4e5f7f5eb85e89/testFamily: creating writer at 1733085368681Flushing 1e5a1c461af1d6f06b4e5f7f5eb85e89/testFamily: appending metadata at 1733085368698 (+17 ms)Flushing 1e5a1c461af1d6f06b4e5f7f5eb85e89/testFamily: closing flushed file at 1733085368698Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e4bb2f: reopening flushed file at 1733085369113 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1e5a1c461af1d6f06b4e5f7f5eb85e89 in 444ms, sequenceid=8, compaction requested=false at 1733085369124 (+11 ms)Writing region close event to WAL at 1733085369125 (+1 ms)Running coprocessor post-close hooks at 1733085369131 (+6 ms)Closed at 1733085369131 2024-12-01T20:36:09,131 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733085352835.1e5a1c461af1d6f06b4e5f7f5eb85e89. 2024-12-01T20:36:09,131 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d16d6d9a6d0117a579e46840c314c902, disabling compactions & flushes 2024-12-01T20:36:09,131 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:36:09,131 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:36:09,131 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. after waiting 0 ms 2024-12-01T20:36:09,131 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:36:09,132 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d16d6d9a6d0117a579e46840c314c902 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:09,154 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902/.tmp/testFamily/c781356d81b34d3988146a80bc6ea14c is 61, key is row-0/testFamily:testQualifier/1733085314122/Put/seqid=0 2024-12-01T20:36:09,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741866_1042 (size=5049) 2024-12-01T20:36:09,197 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on a958fa583d96defe82ac625d03bd54ac, d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:36:09,397 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on a958fa583d96defe82ac625d03bd54ac, d16d6d9a6d0117a579e46840c314c902 2024-12-01T20:36:09,561 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902/.tmp/testFamily/c781356d81b34d3988146a80bc6ea14c 2024-12-01T20:36:09,571 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902/.tmp/testFamily/c781356d81b34d3988146a80bc6ea14c as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902/testFamily/c781356d81b34d3988146a80bc6ea14c 2024-12-01T20:36:09,579 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902/testFamily/c781356d81b34d3988146a80bc6ea14c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:09,581 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d16d6d9a6d0117a579e46840c314c902 in 450ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:09,588 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/d16d6d9a6d0117a579e46840c314c902/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:09,589 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:36:09,589 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d16d6d9a6d0117a579e46840c314c902: Waiting for close lock at 1733085369131Running coprocessor pre-close hooks at 1733085369131Disabling compacts and flushes for region at 1733085369131Disabling writes for close at 1733085369131Obtaining lock to block concurrent updates at 1733085369132 (+1 ms)Preparing flush snapshotting stores in d16d6d9a6d0117a579e46840c314c902 at 1733085369132Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085369132Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. at 1733085369133 (+1 ms)Flushing d16d6d9a6d0117a579e46840c314c902/testFamily: creating writer at 1733085369133Flushing d16d6d9a6d0117a579e46840c314c902/testFamily: appending metadata at 1733085369154 (+21 ms)Flushing d16d6d9a6d0117a579e46840c314c902/testFamily: closing flushed file at 1733085369154Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53cd7636: reopening flushed file at 1733085369569 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d16d6d9a6d0117a579e46840c314c902 in 450ms, sequenceid=8, compaction requested=false at 1733085369581 (+12 ms)Writing region close event to WAL at 1733085369582 (+1 ms)Running coprocessor post-close hooks at 1733085369589 (+7 ms)Closed at 1733085369589 2024-12-01T20:36:09,590 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733085313036.d16d6d9a6d0117a579e46840c314c902. 2024-12-01T20:36:09,590 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a958fa583d96defe82ac625d03bd54ac, disabling compactions & flushes 2024-12-01T20:36:09,590 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:36:09,590 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:36:09,590 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. after waiting 0 ms 2024-12-01T20:36:09,590 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:36:09,590 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a958fa583d96defe82ac625d03bd54ac 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-01T20:36:09,597 DEBUG [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1351): Waiting on a958fa583d96defe82ac625d03bd54ac 2024-12-01T20:36:09,612 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac/.tmp/testFamily/ed24022b261944a6ade1abbd77f06b04 is 61, key is row-0/testFamily:testQualifier/1733085331459/Put/seqid=0 2024-12-01T20:36:09,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741867_1043 (size=5049) 2024-12-01T20:36:09,619 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac/.tmp/testFamily/ed24022b261944a6ade1abbd77f06b04 2024-12-01T20:36:09,628 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac/.tmp/testFamily/ed24022b261944a6ade1abbd77f06b04 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac/testFamily/ed24022b261944a6ade1abbd77f06b04 2024-12-01T20:36:09,637 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac/testFamily/ed24022b261944a6ade1abbd77f06b04, entries=4, sequenceid=8, filesize=4.9 K 2024-12-01T20:36:09,639 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a958fa583d96defe82ac625d03bd54ac in 48ms, sequenceid=8, compaction requested=false 2024-12-01T20:36:09,644 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/data/default/testRetryOutOfOrderScannerNextException/a958fa583d96defe82ac625d03bd54ac/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-01T20:36:09,644 INFO [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:36:09,645 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a958fa583d96defe82ac625d03bd54ac: Waiting for close lock at 1733085369590Running coprocessor pre-close hooks at 1733085369590Disabling compacts and flushes for region at 1733085369590Disabling writes for close at 1733085369590Obtaining lock to block concurrent updates at 1733085369590Preparing flush snapshotting stores in a958fa583d96defe82ac625d03bd54ac at 1733085369590Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733085369590Flushing stores of testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. at 1733085369591 (+1 ms)Flushing a958fa583d96defe82ac625d03bd54ac/testFamily: creating writer at 1733085369591Flushing a958fa583d96defe82ac625d03bd54ac/testFamily: appending metadata at 1733085369612 (+21 ms)Flushing a958fa583d96defe82ac625d03bd54ac/testFamily: closing flushed file at 1733085369612Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bc11523: reopening flushed file at 1733085369627 (+15 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a958fa583d96defe82ac625d03bd54ac in 48ms, sequenceid=8, compaction requested=false at 1733085369639 (+12 ms)Writing region close event to WAL at 1733085369639Running coprocessor post-close hooks at 1733085369644 (+5 ms)Closed at 1733085369644 2024-12-01T20:36:09,645 DEBUG [RS_CLOSE_REGION-regionserver/2140e1fa03ff:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733085328415.a958fa583d96defe82ac625d03bd54ac. 2024-12-01T20:36:09,798 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(976): stopping server 2140e1fa03ff,38225,1733085248936; all regions closed. 2024-12-01T20:36:09,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741834_1010 (size=19479) 2024-12-01T20:36:09,810 DEBUG [RS:0;2140e1fa03ff:38225 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/oldWALs 2024-12-01T20:36:09,810 INFO [RS:0;2140e1fa03ff:38225 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2140e1fa03ff%2C38225%2C1733085248936.meta:.meta(num 1733085312403) 2024-12-01T20:36:09,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741833_1009 (size=20031) 2024-12-01T20:36:09,813 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/WALs/2140e1fa03ff,38225,1733085248936/2140e1fa03ff%2C38225%2C1733085248936.1733085311989 not finished, retry = 0 2024-12-01T20:36:09,917 DEBUG [RS:0;2140e1fa03ff:38225 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/oldWALs 2024-12-01T20:36:09,917 INFO [RS:0;2140e1fa03ff:38225 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2140e1fa03ff%2C38225%2C1733085248936:(num 1733085311989) 2024-12-01T20:36:09,917 DEBUG [RS:0;2140e1fa03ff:38225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-01T20:36:09,917 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T20:36:09,918 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T20:36:09,918 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.ChoreService(370): Chore service for: regionserver/2140e1fa03ff:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-01T20:36:09,918 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T20:36:09,918 INFO [regionserver/2140e1fa03ff:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T20:36:09,919 INFO [RS:0;2140e1fa03ff:38225 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38225 2024-12-01T20:36:09,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2140e1fa03ff,38225,1733085248936 2024-12-01T20:36:09,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-01T20:36:09,924 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T20:36:09,927 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2140e1fa03ff,38225,1733085248936] 2024-12-01T20:36:09,928 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2140e1fa03ff,38225,1733085248936 already deleted, retry=false 2024-12-01T20:36:09,928 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2140e1fa03ff,38225,1733085248936 expired; onlineServers=0 2024-12-01T20:36:09,928 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2140e1fa03ff,33377,1733085247739' ***** 2024-12-01T20:36:09,928 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-01T20:36:09,929 INFO [M:0;2140e1fa03ff:33377 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-01T20:36:09,929 INFO [M:0;2140e1fa03ff:33377 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-01T20:36:09,929 DEBUG [M:0;2140e1fa03ff:33377 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-01T20:36:09,929 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-01T20:36:09,929 DEBUG [M:0;2140e1fa03ff:33377 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-01T20:36:09,929 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster-HFileCleaner.large.0-1733085251935 {}] cleaner.HFileCleaner(306): Exit Thread[master/2140e1fa03ff:0:becomeActiveMaster-HFileCleaner.large.0-1733085251935,5,FailOnTimeoutGroup] 2024-12-01T20:36:09,929 INFO [M:0;2140e1fa03ff:33377 {}] hbase.ChoreService(370): Chore service for: master/2140e1fa03ff:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-01T20:36:09,929 INFO [M:0;2140e1fa03ff:33377 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-01T20:36:09,929 DEBUG [master/2140e1fa03ff:0:becomeActiveMaster-HFileCleaner.small.0-1733085251940 {}] cleaner.HFileCleaner(306): Exit Thread[master/2140e1fa03ff:0:becomeActiveMaster-HFileCleaner.small.0-1733085251940,5,FailOnTimeoutGroup] 2024-12-01T20:36:09,929 DEBUG [M:0;2140e1fa03ff:33377 {}] master.HMaster(1795): Stopping service threads 2024-12-01T20:36:09,930 INFO [M:0;2140e1fa03ff:33377 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-01T20:36:09,930 INFO [M:0;2140e1fa03ff:33377 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-01T20:36:09,930 INFO [M:0;2140e1fa03ff:33377 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-01T20:36:09,931 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-01T20:36:09,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-01T20:36:09,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-01T20:36:09,932 DEBUG [M:0;2140e1fa03ff:33377 {}] zookeeper.ZKUtil(347): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-01T20:36:09,932 WARN [M:0;2140e1fa03ff:33377 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-01T20:36:09,933 INFO [M:0;2140e1fa03ff:33377 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/.lastflushedseqids 2024-12-01T20:36:09,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741868_1044 (size=460) 2024-12-01T20:36:09,947 INFO [M:0;2140e1fa03ff:33377 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-01T20:36:09,948 INFO [M:0;2140e1fa03ff:33377 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-01T20:36:09,948 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-01T20:36:09,948 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:36:09,948 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:36:09,948 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-01T20:36:09,948 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:36:09,948 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.95 KB heapSize=190.50 KB 2024-12-01T20:36:09,972 DEBUG [M:0;2140e1fa03ff:33377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51c395b3a8e54888960427f7c7fd9ab8 is 82, key is hbase:meta,,1/info:regioninfo/1733085312522/Put/seqid=0 2024-12-01T20:36:09,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741869_1045 (size=5672) 2024-12-01T20:36:09,979 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51c395b3a8e54888960427f7c7fd9ab8 2024-12-01T20:36:10,003 INFO [regionserver/2140e1fa03ff:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-01T20:36:10,011 DEBUG [M:0;2140e1fa03ff:33377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/49a2ffbdb9a442b2a81af83b6453eb19 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733085313931/Put/seqid=0 2024-12-01T20:36:10,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741870_1046 (size=14081) 2024-12-01T20:36:10,023 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.40 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/49a2ffbdb9a442b2a81af83b6453eb19 2024-12-01T20:36:10,028 INFO [RS:0;2140e1fa03ff:38225 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T20:36:10,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T20:36:10,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38225-0x1016fbc20130001, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T20:36:10,028 INFO [RS:0;2140e1fa03ff:38225 {}] regionserver.HRegionServer(1031): Exiting; stopping=2140e1fa03ff,38225,1733085248936; zookeeper connection closed. 2024-12-01T20:36:10,028 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@71ff9859 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@71ff9859 2024-12-01T20:36:10,029 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-01T20:36:10,032 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 49a2ffbdb9a442b2a81af83b6453eb19 2024-12-01T20:36:10,057 DEBUG [M:0;2140e1fa03ff:33377 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8af677d726fc48c28c56cf36cd9923e5 is 69, key is 2140e1fa03ff,38225,1733085248936/rs:state/1733085281842/Put/seqid=0 2024-12-01T20:36:10,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741871_1047 (size=5156) 2024-12-01T20:36:10,464 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8af677d726fc48c28c56cf36cd9923e5 2024-12-01T20:36:10,472 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/51c395b3a8e54888960427f7c7fd9ab8 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/51c395b3a8e54888960427f7c7fd9ab8 2024-12-01T20:36:10,479 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/51c395b3a8e54888960427f7c7fd9ab8, entries=8, sequenceid=312, filesize=5.5 K 2024-12-01T20:36:10,480 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/49a2ffbdb9a442b2a81af83b6453eb19 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/49a2ffbdb9a442b2a81af83b6453eb19 2024-12-01T20:36:10,487 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 49a2ffbdb9a442b2a81af83b6453eb19 2024-12-01T20:36:10,487 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/49a2ffbdb9a442b2a81af83b6453eb19, entries=33, sequenceid=312, filesize=13.8 K 2024-12-01T20:36:10,488 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8af677d726fc48c28c56cf36cd9923e5 as hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8af677d726fc48c28c56cf36cd9923e5 2024-12-01T20:36:10,496 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40247/user/jenkins/test-data/04c7ef92-b5c2-90ae-993e-ff95d92a298d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8af677d726fc48c28c56cf36cd9923e5, entries=1, sequenceid=312, filesize=5.0 K 2024-12-01T20:36:10,497 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.95 KB/166863, heapSize ~190.20 KB/194768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 549ms, sequenceid=312, compaction requested=false 2024-12-01T20:36:10,498 INFO [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-01T20:36:10,498 DEBUG [M:0;2140e1fa03ff:33377 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733085369948Disabling compacts and flushes for region at 1733085369948Disabling writes for close at 1733085369948Obtaining lock to block concurrent updates at 1733085369948Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733085369948Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166863, getHeapSize=195008, getOffHeapSize=0, getCellsCount=359 at 1733085369949 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733085369949Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733085369949Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733085369971 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733085369971Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733085369987 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733085370010 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733085370010Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733085370032 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733085370056 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733085370057 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36672d04: reopening flushed file at 1733085370471 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65647050: reopening flushed file at 1733085370479 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@705607a0: reopening flushed file at 1733085370487 (+8 ms)Finished flush of dataSize ~162.95 KB/166863, heapSize ~190.20 KB/194768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 549ms, sequenceid=312, compaction requested=false at 1733085370497 (+10 ms)Writing region close event to WAL at 1733085370498 (+1 ms)Closed at 1733085370498 2024-12-01T20:36:10,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741830_1006 (size=187750) 2024-12-01T20:36:10,503 INFO [M:0;2140e1fa03ff:33377 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-01T20:36:10,503 INFO [M:0;2140e1fa03ff:33377 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33377 2024-12-01T20:36:10,503 INFO [M:0;2140e1fa03ff:33377 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-01T20:36:10,503 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-01T20:36:10,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T20:36:10,609 INFO [M:0;2140e1fa03ff:33377 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-01T20:36:10,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33377-0x1016fbc20130000, quorum=127.0.0.1:56459, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-01T20:36:10,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@258822a0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-01T20:36:10,619 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@514e86a2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T20:36:10,620 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:36:10,620 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c1ee62c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T20:36:10,620 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d304465{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/hadoop.log.dir/,STOPPED} 2024-12-01T20:36:10,623 WARN [BP-1961397349-172.17.0.2-1733085244688 heartbeating to localhost/127.0.0.1:40247 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-01T20:36:10,623 WARN [BP-1961397349-172.17.0.2-1733085244688 heartbeating to localhost/127.0.0.1:40247 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1961397349-172.17.0.2-1733085244688 (Datanode Uuid fb7319d7-5440-41d8-9628-ddc680e7b2f0) service to localhost/127.0.0.1:40247 2024-12-01T20:36:10,625 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/cluster_5ee29cae-c15d-e677-3d2e-70d984d05d2a/data/data1/current/BP-1961397349-172.17.0.2-1733085244688 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T20:36:10,626 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/cluster_5ee29cae-c15d-e677-3d2e-70d984d05d2a/data/data2/current/BP-1961397349-172.17.0.2-1733085244688 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-01T20:36:10,627 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-01T20:36:10,627 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-01T20:36:10,628 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-01T20:36:10,641 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@59bae7b3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-01T20:36:10,642 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f237650{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-01T20:36:10,642 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-01T20:36:10,642 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45f63f41{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-01T20:36:10,642 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@660c56f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ec046179-e6a7-b3c9-0a1e-a7e007888f25/hadoop.log.dir/,STOPPED} 2024-12-01T20:36:10,654 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-01T20:36:10,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down