2024-11-28 10:40:25,463 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-28 10:40:25,483 main DEBUG Took 0.012725 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 10:40:25,484 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 10:40:25,484 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 10:40:25,485 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 10:40:25,487 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,494 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 10:40:25,508 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,510 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,512 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,514 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,515 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,517 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,517 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,518 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,519 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,519 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,520 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,521 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,521 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 10:40:25,522 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,522 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 10:40:25,524 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 10:40:25,526 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 10:40:25,528 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 10:40:25,528 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 10:40:25,529 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 10:40:25,530 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 10:40:25,537 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 10:40:25,539 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 10:40:25,541 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 10:40:25,541 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 10:40:25,541 main DEBUG createAppenders(={Console}) 2024-11-28 10:40:25,542 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-28 10:40:25,542 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-28 10:40:25,543 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-28 10:40:25,543 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 10:40:25,543 main DEBUG OutputStream closed 2024-11-28 10:40:25,544 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 10:40:25,544 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 10:40:25,544 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-28 10:40:25,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 10:40:25,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 10:40:25,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 10:40:25,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 10:40:25,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 10:40:25,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 10:40:25,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 10:40:25,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 10:40:25,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 10:40:25,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 10:40:25,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 10:40:25,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 10:40:25,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 10:40:25,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 10:40:25,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 10:40:25,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 10:40:25,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 10:40:25,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 10:40:25,632 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 10:40:25,632 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-28 10:40:25,633 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 10:40:25,634 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-28T10:40:25,645 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-28 10:40:25,648 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 10:40:25,648 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T10:40:25,889 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99 2024-11-28T10:40:25,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T10:40:25,930 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/cluster_148ccf20-f6ae-26ef-ff5e-8273dc2c0562, deleteOnExit=true 2024-11-28T10:40:25,930 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T10:40:25,932 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/test.cache.data in system properties and HBase conf 2024-11-28T10:40:25,933 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T10:40:25,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/hadoop.log.dir in system properties and HBase conf 2024-11-28T10:40:25,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T10:40:25,935 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T10:40:25,936 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T10:40:26,025 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T10:40:26,109 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T10:40:26,112 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T10:40:26,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T10:40:26,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T10:40:26,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T10:40:26,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T10:40:26,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T10:40:26,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T10:40:26,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T10:40:26,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T10:40:26,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/nfs.dump.dir in system properties and HBase conf 2024-11-28T10:40:26,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/java.io.tmpdir in system properties and HBase conf 2024-11-28T10:40:26,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T10:40:26,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T10:40:26,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T10:40:27,163 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T10:40:27,243 INFO [Time-limited test {}] log.Log(170): Logging initialized @2474ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T10:40:27,326 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T10:40:27,387 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T10:40:27,409 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T10:40:27,409 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T10:40:27,410 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T10:40:27,422 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T10:40:27,425 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@660c56f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/hadoop.log.dir/,AVAILABLE} 2024-11-28T10:40:27,427 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@45f63f41{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T10:40:27,614 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@59bae7b3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/java.io.tmpdir/jetty-localhost-41635-hadoop-hdfs-3_4_1-tests_jar-_-any-7540634730971518418/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T10:40:27,623 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f237650{HTTP/1.1, (http/1.1)}{localhost:41635} 2024-11-28T10:40:27,624 INFO [Time-limited test {}] server.Server(415): Started @2856ms 2024-11-28T10:40:28,150 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T10:40:28,157 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T10:40:28,158 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T10:40:28,159 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T10:40:28,159 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T10:40:28,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d304465{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/hadoop.log.dir/,AVAILABLE} 2024-11-28T10:40:28,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c1ee62c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T10:40:28,258 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@258822a0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/java.io.tmpdir/jetty-localhost-41555-hadoop-hdfs-3_4_1-tests_jar-_-any-18417897437284347155/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T10:40:28,259 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@514e86a2{HTTP/1.1, (http/1.1)}{localhost:41555} 2024-11-28T10:40:28,259 INFO [Time-limited test {}] server.Server(415): Started @3491ms 2024-11-28T10:40:28,305 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T10:40:29,363 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/cluster_148ccf20-f6ae-26ef-ff5e-8273dc2c0562/data/data1/current/BP-1945411675-172.17.0.2-1732790426683/current, will proceed with Du for space computation calculation, 2024-11-28T10:40:29,363 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/cluster_148ccf20-f6ae-26ef-ff5e-8273dc2c0562/data/data2/current/BP-1945411675-172.17.0.2-1732790426683/current, will proceed with Du for space computation calculation, 2024-11-28T10:40:29,394 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T10:40:29,441 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa4c90e48744345a7 with lease ID 0xdbf23e7eb0e581f1: Processing first storage report for DS-8803b344-a872-4f21-a2de-11fcefdf066b from datanode DatanodeRegistration(127.0.0.1:38247, datanodeUuid=11f46c8d-8f38-42be-87d8-261561bf6700, infoPort=40561, infoSecurePort=0, ipcPort=46827, storageInfo=lv=-57;cid=testClusterID;nsid=1568282892;c=1732790426683) 2024-11-28T10:40:29,443 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa4c90e48744345a7 with lease ID 0xdbf23e7eb0e581f1: from storage DS-8803b344-a872-4f21-a2de-11fcefdf066b node DatanodeRegistration(127.0.0.1:38247, datanodeUuid=11f46c8d-8f38-42be-87d8-261561bf6700, infoPort=40561, infoSecurePort=0, ipcPort=46827, storageInfo=lv=-57;cid=testClusterID;nsid=1568282892;c=1732790426683), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-28T10:40:29,443 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa4c90e48744345a7 with lease ID 0xdbf23e7eb0e581f1: Processing first storage report for DS-7f0b55e8-b49a-4c47-a2bb-c350908109ad from datanode DatanodeRegistration(127.0.0.1:38247, datanodeUuid=11f46c8d-8f38-42be-87d8-261561bf6700, infoPort=40561, infoSecurePort=0, ipcPort=46827, storageInfo=lv=-57;cid=testClusterID;nsid=1568282892;c=1732790426683) 2024-11-28T10:40:29,444 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa4c90e48744345a7 with lease ID 0xdbf23e7eb0e581f1: from storage DS-7f0b55e8-b49a-4c47-a2bb-c350908109ad node DatanodeRegistration(127.0.0.1:38247, datanodeUuid=11f46c8d-8f38-42be-87d8-261561bf6700, infoPort=40561, infoSecurePort=0, ipcPort=46827, storageInfo=lv=-57;cid=testClusterID;nsid=1568282892;c=1732790426683), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T10:40:29,527 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99 2024-11-28T10:40:29,600 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/cluster_148ccf20-f6ae-26ef-ff5e-8273dc2c0562/zookeeper_0, clientPort=62053, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/cluster_148ccf20-f6ae-26ef-ff5e-8273dc2c0562/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/cluster_148ccf20-f6ae-26ef-ff5e-8273dc2c0562/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T10:40:29,610 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62053 2024-11-28T10:40:29,625 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T10:40:29,628 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T10:40:29,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741825_1001 (size=7) 2024-11-28T10:40:30,242 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 with version=8 2024-11-28T10:40:30,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/hbase-staging 2024-11-28T10:40:30,317 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T10:40:30,524 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e792e7e95410:0 server-side Connection retries=9 2024-11-28T10:40:30,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T10:40:30,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T10:40:30,538 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T10:40:30,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T10:40:30,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T10:40:30,662 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T10:40:30,716 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T10:40:30,724 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T10:40:30,727 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T10:40:30,748 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 112395 (auto-detected) 2024-11-28T10:40:30,749 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T10:40:30,766 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42151 2024-11-28T10:40:30,786 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42151 connecting to ZooKeeper ensemble=127.0.0.1:62053 2024-11-28T10:40:30,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:421510x0, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T10:40:30,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42151-0x1018107a3380000 connected 2024-11-28T10:40:31,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T10:40:31,389 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T10:40:31,401 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T10:40:31,405 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226, hbase.cluster.distributed=false 2024-11-28T10:40:31,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T10:40:31,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42151 2024-11-28T10:40:31,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42151 2024-11-28T10:40:31,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42151 2024-11-28T10:40:31,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42151 2024-11-28T10:40:31,431 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42151 2024-11-28T10:40:31,522 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e792e7e95410:0 server-side Connection retries=9 2024-11-28T10:40:31,523 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T10:40:31,524 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T10:40:31,524 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T10:40:31,524 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T10:40:31,524 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T10:40:31,526 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T10:40:31,529 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T10:40:31,530 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34367 2024-11-28T10:40:31,532 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34367 connecting to ZooKeeper ensemble=127.0.0.1:62053 2024-11-28T10:40:31,533 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T10:40:31,535 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T10:40:31,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:343670x0, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T10:40:31,573 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:343670x0, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T10:40:31,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34367-0x1018107a3380001 connected 2024-11-28T10:40:31,578 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T10:40:31,588 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T10:40:31,590 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T10:40:31,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T10:40:31,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34367 2024-11-28T10:40:31,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34367 2024-11-28T10:40:31,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34367 2024-11-28T10:40:31,601 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34367 2024-11-28T10:40:31,602 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34367 2024-11-28T10:40:31,615 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e792e7e95410:42151 2024-11-28T10:40:31,616 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e792e7e95410,42151,1732790430369 2024-11-28T10:40:31,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T10:40:31,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T10:40:31,677 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e792e7e95410,42151,1732790430369 2024-11-28T10:40:31,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T10:40:31,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:31,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:31,789 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T10:40:31,792 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e792e7e95410,42151,1732790430369 from backup master directory 2024-11-28T10:40:31,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T10:40:31,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e792e7e95410,42151,1732790430369 2024-11-28T10:40:31,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T10:40:31,849 WARN [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T10:40:31,849 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e792e7e95410,42151,1732790430369 2024-11-28T10:40:31,852 INFO [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T10:40:31,854 INFO [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T10:40:31,909 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/hbase.id] with ID: bea3c8b0-a0f9-4d9f-9722-bf90762c0b2a 2024-11-28T10:40:31,909 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/.tmp/hbase.id 2024-11-28T10:40:31,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741826_1002 (size=42) 2024-11-28T10:40:32,321 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/.tmp/hbase.id]:[hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/hbase.id] 2024-11-28T10:40:32,370 INFO [master/e792e7e95410:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T10:40:32,377 INFO [master/e792e7e95410:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T10:40:32,397 INFO [master/e792e7e95410:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-28T10:40:32,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:32,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:32,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741827_1003 (size=196) 2024-11-28T10:40:32,843 INFO [master/e792e7e95410:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:40:32,846 INFO [master/e792e7e95410:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T10:40:32,859 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T10:40:32,862 INFO [master/e792e7e95410:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T10:40:32,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741828_1004 (size=1189) 2024-11-28T10:40:33,318 INFO [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store 2024-11-28T10:40:33,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741829_1005 (size=34) 2024-11-28T10:40:33,743 INFO [master/e792e7e95410:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T10:40:33,746 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:40:33,747 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T10:40:33,747 INFO [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T10:40:33,748 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T10:40:33,749 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T10:40:33,750 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T10:40:33,750 INFO [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T10:40:33,751 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732790433747Disabling compacts and flushes for region at 1732790433747Disabling writes for close at 1732790433749 (+2 ms)Writing region close event to WAL at 1732790433750 (+1 ms)Closed at 1732790433750 2024-11-28T10:40:33,754 WARN [master/e792e7e95410:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/.initializing 2024-11-28T10:40:33,754 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/WALs/e792e7e95410,42151,1732790430369 2024-11-28T10:40:33,765 INFO [master/e792e7e95410:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T10:40:33,782 INFO [master/e792e7e95410:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e792e7e95410%2C42151%2C1732790430369, suffix=, logDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/WALs/e792e7e95410,42151,1732790430369, archiveDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/oldWALs, maxLogs=10 2024-11-28T10:40:33,806 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/WALs/e792e7e95410,42151,1732790430369/e792e7e95410%2C42151%2C1732790430369.1732790433787, exclude list is [], retry=0 2024-11-28T10:40:33,820 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38247,DS-8803b344-a872-4f21-a2de-11fcefdf066b,DISK] 2024-11-28T10:40:33,823 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T10:40:33,852 INFO [master/e792e7e95410:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/WALs/e792e7e95410,42151,1732790430369/e792e7e95410%2C42151%2C1732790430369.1732790433787 2024-11-28T10:40:33,853 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40561:40561)] 2024-11-28T10:40:33,853 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:40:33,854 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:40:33,856 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,857 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,891 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,914 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T10:40:33,917 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:33,919 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:40:33,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T10:40:33,923 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:33,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:40:33,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T10:40:33,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:33,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:40:33,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T10:40:33,933 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:33,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:40:33,934 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,938 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,939 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,944 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,945 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,948 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T10:40:33,951 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T10:40:33,957 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:40:33,959 INFO [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70341664, jitterRate=0.0481724739074707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T10:40:33,967 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732790433868Initializing all the Stores at 1732790433870 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732790433870Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790433871 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790433871Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790433871Cleaning up temporary data from old regions at 1732790433945 (+74 ms)Region opened successfully at 1732790433967 (+22 ms) 2024-11-28T10:40:33,968 INFO [master/e792e7e95410:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T10:40:34,003 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5272efbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e792e7e95410/172.17.0.2:0 2024-11-28T10:40:34,037 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T10:40:34,048 INFO [master/e792e7e95410:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T10:40:34,048 INFO [master/e792e7e95410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T10:40:34,052 INFO [master/e792e7e95410:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T10:40:34,054 INFO [master/e792e7e95410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T10:40:34,061 INFO [master/e792e7e95410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-28T10:40:34,061 INFO [master/e792e7e95410:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T10:40:34,093 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T10:40:34,104 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T10:40:34,112 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T10:40:34,115 INFO [master/e792e7e95410:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T10:40:34,116 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T10:40:34,122 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T10:40:34,125 INFO [master/e792e7e95410:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T10:40:34,130 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T10:40:34,141 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T10:40:34,143 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T10:40:34,154 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T10:40:34,170 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T10:40:34,182 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T10:40:34,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T10:40:34,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:34,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T10:40:34,197 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:34,200 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e792e7e95410,42151,1732790430369, sessionid=0x1018107a3380000, setting cluster-up flag (Was=false) 2024-11-28T10:40:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:34,301 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T10:40:34,304 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e792e7e95410,42151,1732790430369 2024-11-28T10:40:34,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:34,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:40:34,364 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T10:40:34,366 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e792e7e95410,42151,1732790430369 2024-11-28T10:40:34,371 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T10:40:34,408 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(746): ClusterId : bea3c8b0-a0f9-4d9f-9722-bf90762c0b2a 2024-11-28T10:40:34,412 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T10:40:34,427 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T10:40:34,427 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T10:40:34,437 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T10:40:34,439 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T10:40:34,440 DEBUG [RS:0;e792e7e95410:34367 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a576393, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e792e7e95410/172.17.0.2:0 2024-11-28T10:40:34,448 INFO [master/e792e7e95410:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T10:40:34,456 INFO [master/e792e7e95410:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T10:40:34,459 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e792e7e95410:34367 2024-11-28T10:40:34,463 INFO [RS:0;e792e7e95410:34367 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T10:40:34,464 INFO [RS:0;e792e7e95410:34367 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T10:40:34,464 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T10:40:34,462 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e792e7e95410,42151,1732790430369 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T10:40:34,467 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(2659): reportForDuty to master=e792e7e95410,42151,1732790430369 with port=34367, startcode=1732790431492 2024-11-28T10:40:34,470 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e792e7e95410:0, corePoolSize=5, maxPoolSize=5 2024-11-28T10:40:34,470 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e792e7e95410:0, corePoolSize=5, maxPoolSize=5 2024-11-28T10:40:34,470 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e792e7e95410:0, corePoolSize=5, maxPoolSize=5 2024-11-28T10:40:34,471 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e792e7e95410:0, corePoolSize=5, maxPoolSize=5 2024-11-28T10:40:34,471 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e792e7e95410:0, corePoolSize=10, maxPoolSize=10 2024-11-28T10:40:34,471 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,471 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e792e7e95410:0, corePoolSize=2, maxPoolSize=2 2024-11-28T10:40:34,471 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,473 INFO [master/e792e7e95410:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732790464473 2024-11-28T10:40:34,474 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T10:40:34,475 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T10:40:34,477 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T10:40:34,477 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T10:40:34,479 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T10:40:34,479 DEBUG [RS:0;e792e7e95410:34367 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T10:40:34,479 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T10:40:34,479 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T10:40:34,479 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T10:40:34,480 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,483 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T10:40:34,484 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T10:40:34,485 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T10:40:34,485 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:34,485 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T10:40:34,486 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T10:40:34,487 INFO [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T10:40:34,488 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e792e7e95410:0:becomeActiveMaster-HFileCleaner.large.0-1732790434488,5,FailOnTimeoutGroup] 2024-11-28T10:40:34,489 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e792e7e95410:0:becomeActiveMaster-HFileCleaner.small.0-1732790434488,5,FailOnTimeoutGroup] 2024-11-28T10:40:34,489 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,489 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T10:40:34,491 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,491 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741831_1007 (size=1321) 2024-11-28T10:40:34,515 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40703, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T10:40:34,521 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e792e7e95410,34367,1732790431492 2024-11-28T10:40:34,523 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42151 {}] master.ServerManager(517): Registering regionserver=e792e7e95410,34367,1732790431492 2024-11-28T10:40:34,536 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:40:34,536 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33827 2024-11-28T10:40:34,537 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T10:40:34,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T10:40:34,552 DEBUG [RS:0;e792e7e95410:34367 {}] zookeeper.ZKUtil(111): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e792e7e95410,34367,1732790431492 2024-11-28T10:40:34,552 WARN [RS:0;e792e7e95410:34367 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T10:40:34,552 INFO [RS:0;e792e7e95410:34367 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T10:40:34,553 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/WALs/e792e7e95410,34367,1732790431492 2024-11-28T10:40:34,555 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e792e7e95410,34367,1732790431492] 2024-11-28T10:40:34,577 INFO [RS:0;e792e7e95410:34367 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T10:40:34,593 INFO [RS:0;e792e7e95410:34367 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T10:40:34,598 INFO [RS:0;e792e7e95410:34367 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T10:40:34,598 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,599 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T10:40:34,604 INFO [RS:0;e792e7e95410:34367 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T10:40:34,605 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,606 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,606 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,606 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,606 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,606 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,606 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e792e7e95410:0, corePoolSize=2, maxPoolSize=2 2024-11-28T10:40:34,606 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,607 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,607 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,607 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,607 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,607 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e792e7e95410:0, corePoolSize=1, maxPoolSize=1 2024-11-28T10:40:34,607 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e792e7e95410:0, corePoolSize=3, maxPoolSize=3 2024-11-28T10:40:34,607 DEBUG [RS:0;e792e7e95410:34367 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e792e7e95410:0, corePoolSize=3, maxPoolSize=3 2024-11-28T10:40:34,608 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,608 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,609 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,609 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,609 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,609 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=e792e7e95410,34367,1732790431492-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T10:40:34,627 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T10:40:34,628 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=e792e7e95410,34367,1732790431492-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,629 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,629 INFO [RS:0;e792e7e95410:34367 {}] regionserver.Replication(171): e792e7e95410,34367,1732790431492 started 2024-11-28T10:40:34,645 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T10:40:34,645 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1482): Serving as e792e7e95410,34367,1732790431492, RpcServer on e792e7e95410/172.17.0.2:34367, sessionid=0x1018107a3380001 2024-11-28T10:40:34,646 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T10:40:34,646 DEBUG [RS:0;e792e7e95410:34367 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e792e7e95410,34367,1732790431492 2024-11-28T10:40:34,646 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e792e7e95410,34367,1732790431492' 2024-11-28T10:40:34,646 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T10:40:34,647 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T10:40:34,648 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T10:40:34,648 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T10:40:34,648 DEBUG [RS:0;e792e7e95410:34367 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e792e7e95410,34367,1732790431492 2024-11-28T10:40:34,648 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e792e7e95410,34367,1732790431492' 2024-11-28T10:40:34,648 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T10:40:34,649 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T10:40:34,650 DEBUG [RS:0;e792e7e95410:34367 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T10:40:34,650 INFO [RS:0;e792e7e95410:34367 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T10:40:34,650 INFO [RS:0;e792e7e95410:34367 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T10:40:34,898 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T10:40:34,899 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:40:34,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741832_1008 (size=32) 2024-11-28T10:40:35,310 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:40:35,313 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T10:40:35,317 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T10:40:35,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:35,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:40:35,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T10:40:35,322 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T10:40:35,322 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:35,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:40:35,323 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T10:40:35,326 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T10:40:35,326 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:35,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:40:35,328 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T10:40:35,330 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T10:40:35,330 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:40:35,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:40:35,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T10:40:35,333 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740 2024-11-28T10:40:35,334 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740 2024-11-28T10:40:35,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T10:40:35,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T10:40:35,338 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T10:40:35,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T10:40:35,344 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:40:35,345 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74979794, jitterRate=0.11728599667549133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T10:40:35,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732790435311Initializing all the Stores at 1732790435313 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732790435313Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732790435313Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790435313Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732790435313Cleaning up temporary data from old regions at 1732790435337 (+24 ms)Region opened successfully at 1732790435347 (+10 ms) 2024-11-28T10:40:35,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T10:40:35,348 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T10:40:35,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T10:40:35,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T10:40:35,348 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T10:40:35,349 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T10:40:35,349 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732790435348Disabling compacts and flushes for region at 1732790435348Disabling writes for close at 1732790435348Writing region close event to WAL at 1732790435349 (+1 ms)Closed at 1732790435349 2024-11-28T10:40:35,352 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T10:40:35,353 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T10:40:35,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T10:40:35,371 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T10:40:35,374 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T10:40:35,526 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:36,527 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:37,527 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:38,528 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:39,529 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:40,529 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:40,713 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T10:40:40,714 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T10:40:41,530 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:42,531 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:43,532 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:44,533 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:45,534 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:46,534 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:47,535 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:48,536 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:49,537 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:50,537 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:51,538 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:52,539 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:53,540 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:54,541 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:55,542 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:56,543 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:57,543 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:58,544 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:40:59,523 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T10:40:59,545 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:41:00,546 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:41:01,546 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:41:02,547 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:41:03,548 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:41:04,549 WARN [e792e7e95410:42151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T10:41:04,655 INFO [RS:0;e792e7e95410:34367 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T10:41:04,659 INFO [RS:0;e792e7e95410:34367 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e792e7e95410%2C34367%2C1732790431492, suffix=, logDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/WALs/e792e7e95410,34367,1732790431492, archiveDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/oldWALs, maxLogs=32 2024-11-28T10:41:04,676 DEBUG [RS:0;e792e7e95410:34367 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/WALs/e792e7e95410,34367,1732790431492/e792e7e95410%2C34367%2C1732790431492.1732790464662, exclude list is [], retry=0 2024-11-28T10:41:04,680 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38247,DS-8803b344-a872-4f21-a2de-11fcefdf066b,DISK] 2024-11-28T10:41:04,683 INFO [RS:0;e792e7e95410:34367 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/WALs/e792e7e95410,34367,1732790431492/e792e7e95410%2C34367%2C1732790431492.1732790464662 2024-11-28T10:41:04,683 DEBUG [RS:0;e792e7e95410:34367 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40561:40561)] 2024-11-28T10:41:04,805 DEBUG [e792e7e95410:42151 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T10:41:04,819 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:04,824 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e792e7e95410,34367,1732790431492, state=OPENING 2024-11-28T10:41:04,846 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T10:41:04,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:41:04,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:41:04,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T10:41:04,859 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T10:41:04,861 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T10:41:04,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:05,052 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T10:41:05,055 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39775, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T10:41:05,066 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T10:41:05,066 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T10:41:05,067 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T10:41:05,070 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e792e7e95410%2C34367%2C1732790431492.meta, suffix=.meta, logDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/WALs/e792e7e95410,34367,1732790431492, archiveDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/oldWALs, maxLogs=32 2024-11-28T10:41:05,085 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/WALs/e792e7e95410,34367,1732790431492/e792e7e95410%2C34367%2C1732790431492.meta.1732790465071.meta, exclude list is [], retry=0 2024-11-28T10:41:05,088 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38247,DS-8803b344-a872-4f21-a2de-11fcefdf066b,DISK] 2024-11-28T10:41:05,091 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/WALs/e792e7e95410,34367,1732790431492/e792e7e95410%2C34367%2C1732790431492.meta.1732790465071.meta 2024-11-28T10:41:05,092 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40561:40561)] 2024-11-28T10:41:05,092 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:05,094 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T10:41:05,096 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T10:41:05,101 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T10:41:05,104 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T10:41:05,105 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:05,105 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T10:41:05,105 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T10:41:05,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T10:41:05,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T10:41:05,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:05,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:41:05,111 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T10:41:05,112 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T10:41:05,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:05,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:41:05,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T10:41:05,115 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T10:41:05,115 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:05,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:41:05,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T10:41:05,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T10:41:05,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:05,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T10:41:05,119 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T10:41:05,120 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740 2024-11-28T10:41:05,123 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740 2024-11-28T10:41:05,126 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T10:41:05,126 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T10:41:05,127 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T10:41:05,131 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T10:41:05,133 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65127355, jitterRate=-0.029526785016059875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T10:41:05,133 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T10:41:05,135 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732790465106Writing region info on filesystem at 1732790465106Initializing all the Stores at 1732790465107 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732790465108 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732790465108Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790465108Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732790465108Cleaning up temporary data from old regions at 1732790465126 (+18 ms)Running coprocessor post-open hooks at 1732790465133 (+7 ms)Region opened successfully at 1732790465135 (+2 ms) 2024-11-28T10:41:05,143 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732790465043 2024-11-28T10:41:05,153 DEBUG [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T10:41:05,154 INFO [RS_OPEN_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T10:41:05,156 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:05,159 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e792e7e95410,34367,1732790431492, state=OPEN 2024-11-28T10:41:05,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T10:41:05,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T10:41:05,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T10:41:05,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T10:41:05,198 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e792e7e95410,34367,1732790431492 2024-11-28T10:41:05,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T10:41:05,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e792e7e95410,34367,1732790431492 in 335 msec 2024-11-28T10:41:05,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T10:41:05,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8490 sec 2024-11-28T10:41:05,217 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T10:41:05,217 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T10:41:05,235 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T10:41:05,236 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e792e7e95410,34367,1732790431492, seqNum=-1] 2024-11-28T10:41:05,254 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T10:41:05,256 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33217, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T10:41:05,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.8750 sec 2024-11-28T10:41:05,276 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732790465276, completionTime=-1 2024-11-28T10:41:05,279 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T10:41:05,279 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T10:41:05,302 INFO [master/e792e7e95410:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T10:41:05,302 INFO [master/e792e7e95410:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732790525302 2024-11-28T10:41:05,303 INFO [master/e792e7e95410:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732790585303 2024-11-28T10:41:05,303 INFO [master/e792e7e95410:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-28T10:41:05,304 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e792e7e95410,42151,1732790430369-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T10:41:05,305 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e792e7e95410,42151,1732790430369-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T10:41:05,305 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e792e7e95410,42151,1732790430369-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T10:41:05,306 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e792e7e95410:42151, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T10:41:05,306 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T10:41:05,307 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T10:41:05,312 DEBUG [master/e792e7e95410:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T10:41:05,335 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.486sec 2024-11-28T10:41:05,337 INFO [master/e792e7e95410:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T10:41:05,339 INFO [master/e792e7e95410:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T10:41:05,340 INFO [master/e792e7e95410:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T10:41:05,341 INFO [master/e792e7e95410:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T10:41:05,341 INFO [master/e792e7e95410:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T10:41:05,343 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e792e7e95410,42151,1732790430369-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T10:41:05,344 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e792e7e95410,42151,1732790430369-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T10:41:05,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fc30bac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T10:41:05,353 DEBUG [master/e792e7e95410:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T10:41:05,354 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T10:41:05,354 INFO [master/e792e7e95410:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e792e7e95410,42151,1732790430369-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T10:41:05,355 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T10:41:05,355 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T10:41:05,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e792e7e95410,42151,-1 for getting cluster id 2024-11-28T10:41:05,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T10:41:05,366 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bea3c8b0-a0f9-4d9f-9722-bf90762c0b2a' 2024-11-28T10:41:05,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T10:41:05,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bea3c8b0-a0f9-4d9f-9722-bf90762c0b2a" 2024-11-28T10:41:05,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34742aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T10:41:05,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e792e7e95410,42151,-1] 2024-11-28T10:41:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T10:41:05,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:41:05,375 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T10:41:05,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf9fbe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T10:41:05,378 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T10:41:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e792e7e95410,34367,1732790431492, seqNum=-1] 2024-11-28T10:41:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T10:41:05,390 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T10:41:05,412 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e792e7e95410,42151,1732790430369 2024-11-28T10:41:05,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1986eda2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T10:41:05,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e792e7e95410,42151,-1 for getting cluster id 2024-11-28T10:41:05,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T10:41:05,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bea3c8b0-a0f9-4d9f-9722-bf90762c0b2a' 2024-11-28T10:41:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T10:41:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bea3c8b0-a0f9-4d9f-9722-bf90762c0b2a" 2024-11-28T10:41:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7a2feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T10:41:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e792e7e95410,42151,-1] 2024-11-28T10:41:05,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T10:41:05,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:41:05,422 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T10:41:05,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65e44f4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T10:41:05,436 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=4873 2024-11-28T10:41:05,445 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T10:41:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e792e7e95410,42151,1732790430369 2024-11-28T10:41:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55a08759 2024-11-28T10:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T10:41:05,475 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T10:41:05,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-28T10:41:05,497 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:05,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-28T10:41:05,501 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:05,504 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T10:41:05,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741835_1011 (size=428) 2024-11-28T10:41:05,523 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e99096e2f5934a127dfc49eb28ac61c6, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:05,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741836_1012 (size=79) 2024-11-28T10:41:05,936 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:05,936 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing e99096e2f5934a127dfc49eb28ac61c6, disabling compactions & flushes 2024-11-28T10:41:05,936 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:41:05,936 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:41:05,936 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. after waiting 0 ms 2024-11-28T10:41:05,936 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:41:05,936 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:41:05,936 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e99096e2f5934a127dfc49eb28ac61c6: Waiting for close lock at 1732790465936Disabling compacts and flushes for region at 1732790465936Disabling writes for close at 1732790465936Writing region close event to WAL at 1732790465936Closed at 1732790465936 2024-11-28T10:41:05,940 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:05,948 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732790465940"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790465940"}]},"ts":"1732790465940"} 2024-11-28T10:41:05,953 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:05,955 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:05,957 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790465955"}]},"ts":"1732790465955"} 2024-11-28T10:41:05,961 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-28T10:41:05,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e99096e2f5934a127dfc49eb28ac61c6, ASSIGN}] 2024-11-28T10:41:05,966 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e99096e2f5934a127dfc49eb28ac61c6, ASSIGN 2024-11-28T10:41:05,968 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e99096e2f5934a127dfc49eb28ac61c6, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:06,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e99096e2f5934a127dfc49eb28ac61c6, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:06,126 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e99096e2f5934a127dfc49eb28ac61c6, ASSIGN because future has completed 2024-11-28T10:41:06,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e99096e2f5934a127dfc49eb28ac61c6, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:06,299 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:41:06,299 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e99096e2f5934a127dfc49eb28ac61c6, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:06,300 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,300 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:06,300 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,300 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,303 INFO [StoreOpener-e99096e2f5934a127dfc49eb28ac61c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,305 INFO [StoreOpener-e99096e2f5934a127dfc49eb28ac61c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e99096e2f5934a127dfc49eb28ac61c6 columnFamilyName testFamily 2024-11-28T10:41:06,305 DEBUG [StoreOpener-e99096e2f5934a127dfc49eb28ac61c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:06,306 INFO [StoreOpener-e99096e2f5934a127dfc49eb28ac61c6-1 {}] regionserver.HStore(327): Store=e99096e2f5934a127dfc49eb28ac61c6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:06,306 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,308 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,308 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,309 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,309 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,312 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,315 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:06,316 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e99096e2f5934a127dfc49eb28ac61c6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61236825, jitterRate=-0.08750019967556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:06,316 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:41:06,317 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e99096e2f5934a127dfc49eb28ac61c6: Running coprocessor pre-open hook at 1732790466300Writing region info on filesystem at 1732790466300Initializing all the Stores at 1732790466302 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790466302Cleaning up temporary data from old regions at 1732790466309 (+7 ms)Running coprocessor post-open hooks at 1732790466316 (+7 ms)Region opened successfully at 1732790466317 (+1 ms) 2024-11-28T10:41:06,319 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6., pid=6, masterSystemTime=1732790466285 2024-11-28T10:41:06,322 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:41:06,323 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:41:06,323 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e99096e2f5934a127dfc49eb28ac61c6, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:06,327 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e99096e2f5934a127dfc49eb28ac61c6, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:06,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T10:41:06,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e99096e2f5934a127dfc49eb28ac61c6, server=e792e7e95410,34367,1732790431492 in 202 msec 2024-11-28T10:41:06,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T10:41:06,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e99096e2f5934a127dfc49eb28ac61c6, ASSIGN in 369 msec 2024-11-28T10:41:06,337 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:06,338 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790466337"}]},"ts":"1732790466337"} 2024-11-28T10:41:06,341 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-28T10:41:06,342 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:06,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 858 msec 2024-11-28T10:41:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T10:41:06,535 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-28T10:41:06,535 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-28T10:41:06,536 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:06,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-28T10:41:06,542 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:06,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-28T10:41:06,545 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T10:41:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e792e7e95410,34367,1732790431492, seqNum=-1] 2024-11-28T10:41:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T10:41:06,550 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T10:41:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:06,574 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:06,578 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-28T10:41:06,583 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-28T10:41:06,588 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-28T10:41:06,591 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-28T10:41:06,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3222361654058549240 2024-11-28T10:41:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41288 deadline: 1732790469590, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T10:41:06,629 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-28T10:41:06,631 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-28T10:41:06,633 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:06,635 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:06,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3222361654058549238 2024-11-28T10:41:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41288 deadline: 1732790469635, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T10:41:06,640 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:06,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3222361654058549237 2024-11-28T10:41:06,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41288 deadline: 1732790469640, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T10:41:06,645 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:06,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3222361654058549236 2024-11-28T10:41:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41288 deadline: 1732790469644, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T10:41:06,650 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:06,660 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e792e7e95410:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_891951882_22 at /127.0.0.1:58446 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=4802 (was 4873) 2024-11-28T10:41:06,669 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=4801 2024-11-28T10:41:06,673 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:06,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-28T10:41:06,676 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:06,677 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:06,677 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-28T10:41:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T10:41:06,678 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:06,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741837_1013 (size=414) 2024-11-28T10:41:07,091 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7b909729cbd1c752cd16aaf58c8e110, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:07,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741838_1014 (size=65) 2024-11-28T10:41:07,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:07,506 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing e7b909729cbd1c752cd16aaf58c8e110, disabling compactions & flushes 2024-11-28T10:41:07,506 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:41:07,506 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:41:07,507 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. after waiting 0 ms 2024-11-28T10:41:07,507 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:41:07,507 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:41:07,508 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7b909729cbd1c752cd16aaf58c8e110: Waiting for close lock at 1732790467505Disabling compacts and flushes for region at 1732790467505Disabling writes for close at 1732790467507 (+2 ms)Writing region close event to WAL at 1732790467507Closed at 1732790467507 2024-11-28T10:41:07,511 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:07,511 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732790467511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790467511"}]},"ts":"1732790467511"} 2024-11-28T10:41:07,514 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:07,516 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:07,516 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790467516"}]},"ts":"1732790467516"} 2024-11-28T10:41:07,519 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-28T10:41:07,519 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e7b909729cbd1c752cd16aaf58c8e110, ASSIGN}] 2024-11-28T10:41:07,522 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e7b909729cbd1c752cd16aaf58c8e110, ASSIGN 2024-11-28T10:41:07,523 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e7b909729cbd1c752cd16aaf58c8e110, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:07,675 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e7b909729cbd1c752cd16aaf58c8e110, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T10:41:07,684 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e7b909729cbd1c752cd16aaf58c8e110, ASSIGN because future has completed 2024-11-28T10:41:07,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7b909729cbd1c752cd16aaf58c8e110, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:07,845 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:41:07,845 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e7b909729cbd1c752cd16aaf58c8e110, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:07,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:07,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,849 INFO [StoreOpener-e7b909729cbd1c752cd16aaf58c8e110-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,852 INFO [StoreOpener-e7b909729cbd1c752cd16aaf58c8e110-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7b909729cbd1c752cd16aaf58c8e110 columnFamilyName testFamily 2024-11-28T10:41:07,852 DEBUG [StoreOpener-e7b909729cbd1c752cd16aaf58c8e110-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:07,853 INFO [StoreOpener-e7b909729cbd1c752cd16aaf58c8e110-1 {}] regionserver.HStore(327): Store=e7b909729cbd1c752cd16aaf58c8e110/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:07,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,855 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,861 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,865 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:07,866 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened e7b909729cbd1c752cd16aaf58c8e110; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61846261, jitterRate=-0.07841889560222626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:07,866 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:41:07,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e7b909729cbd1c752cd16aaf58c8e110: Running coprocessor pre-open hook at 1732790467846Writing region info on filesystem at 1732790467847 (+1 ms)Initializing all the Stores at 1732790467848 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790467848Cleaning up temporary data from old regions at 1732790467857 (+9 ms)Running coprocessor post-open hooks at 1732790467866 (+9 ms)Region opened successfully at 1732790467868 (+2 ms) 2024-11-28T10:41:07,869 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110., pid=9, masterSystemTime=1732790467838 2024-11-28T10:41:07,873 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:41:07,873 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:41:07,874 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=e7b909729cbd1c752cd16aaf58c8e110, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:07,878 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7b909729cbd1c752cd16aaf58c8e110, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:07,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-28T10:41:07,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure e7b909729cbd1c752cd16aaf58c8e110, server=e792e7e95410,34367,1732790431492 in 195 msec 2024-11-28T10:41:07,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T10:41:07,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=e7b909729cbd1c752cd16aaf58c8e110, ASSIGN in 365 msec 2024-11-28T10:41:07,889 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:07,889 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790467889"}]},"ts":"1732790467889"} 2024-11-28T10:41:07,893 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-28T10:41:07,895 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:07,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2220 sec 2024-11-28T10:41:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T10:41:09,696 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-28T10:41:09,696 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-28T10:41:09,697 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:09,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-28T10:41:09,706 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:09,706 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-28T10:41:09,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:09,725 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:09,725 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-28T10:41:09,730 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T10:41:09,732 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-28T10:41:10,659 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:41:10,690 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-28T10:41:10,692 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T10:41:10,693 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-28T10:41:10,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T10:41:10,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T10:41:10,714 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T10:41:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-28T10:41:10,714 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:10,715 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-28T10:41:10,715 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:15,740 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110., hostname=e792e7e95410,34367,1732790431492, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110., hostname=e792e7e95410,34367,1732790431492, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e792e7e95410:34367 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-11-28T10:41:15,740 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110., hostname=e792e7e95410,34367,1732790431492, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6007ms, rpcTimeout=5999ms 2024-11-28T10:41:15,740 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110., hostname=e792e7e95410,34367,1732790431492, seqNum=2 from cache 2024-11-28T10:41:15,748 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=4679 (was 4801) 2024-11-28T10:41:15,756 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=168, ProcessCount=11, AvailableMemoryMB=4678 2024-11-28T10:41:15,759 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:15,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-28T10:41:15,763 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:15,764 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:15,764 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-28T10:41:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-28T10:41:15,765 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:15,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741839_1015 (size=412) 2024-11-28T10:41:16,177 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e7e15bd2cf2816daa3a7922c2d73578c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:16,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741840_1016 (size=63) 2024-11-28T10:41:16,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-28T10:41:16,589 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:16,590 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing e7e15bd2cf2816daa3a7922c2d73578c, disabling compactions & flushes 2024-11-28T10:41:16,590 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:41:16,590 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:41:16,590 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. after waiting 0 ms 2024-11-28T10:41:16,590 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:41:16,590 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:41:16,591 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e7e15bd2cf2816daa3a7922c2d73578c: Waiting for close lock at 1732790476590Disabling compacts and flushes for region at 1732790476590Disabling writes for close at 1732790476590Writing region close event to WAL at 1732790476590Closed at 1732790476590 2024-11-28T10:41:16,595 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:16,596 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732790476595"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790476595"}]},"ts":"1732790476595"} 2024-11-28T10:41:16,601 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:16,602 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:16,602 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790476602"}]},"ts":"1732790476602"} 2024-11-28T10:41:16,604 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-28T10:41:16,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=e7e15bd2cf2816daa3a7922c2d73578c, ASSIGN}] 2024-11-28T10:41:16,606 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=e7e15bd2cf2816daa3a7922c2d73578c, ASSIGN 2024-11-28T10:41:16,608 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=e7e15bd2cf2816daa3a7922c2d73578c, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:16,759 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=e7e15bd2cf2816daa3a7922c2d73578c, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:16,768 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=e7e15bd2cf2816daa3a7922c2d73578c, ASSIGN because future has completed 2024-11-28T10:41:16,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7e15bd2cf2816daa3a7922c2d73578c, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:16,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-28T10:41:16,930 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:41:16,930 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => e7e15bd2cf2816daa3a7922c2d73578c, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:16,931 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,931 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:16,931 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,931 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,933 INFO [StoreOpener-e7e15bd2cf2816daa3a7922c2d73578c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,935 INFO [StoreOpener-e7e15bd2cf2816daa3a7922c2d73578c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7e15bd2cf2816daa3a7922c2d73578c columnFamilyName testFamily 2024-11-28T10:41:16,935 DEBUG [StoreOpener-e7e15bd2cf2816daa3a7922c2d73578c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:16,936 INFO [StoreOpener-e7e15bd2cf2816daa3a7922c2d73578c-1 {}] regionserver.HStore(327): Store=e7e15bd2cf2816daa3a7922c2d73578c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:16,936 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,937 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,938 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,939 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,939 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,941 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,944 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:16,945 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened e7e15bd2cf2816daa3a7922c2d73578c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63927990, jitterRate=-0.04739871621131897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:16,945 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:41:16,946 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for e7e15bd2cf2816daa3a7922c2d73578c: Running coprocessor pre-open hook at 1732790476931Writing region info on filesystem at 1732790476931Initializing all the Stores at 1732790476933 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790476933Cleaning up temporary data from old regions at 1732790476939 (+6 ms)Running coprocessor post-open hooks at 1732790476945 (+6 ms)Region opened successfully at 1732790476946 (+1 ms) 2024-11-28T10:41:16,947 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c., pid=12, masterSystemTime=1732790476925 2024-11-28T10:41:16,950 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:41:16,950 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:41:16,951 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=e7e15bd2cf2816daa3a7922c2d73578c, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:16,955 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure e7e15bd2cf2816daa3a7922c2d73578c, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:16,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T10:41:16,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure e7e15bd2cf2816daa3a7922c2d73578c, server=e792e7e95410,34367,1732790431492 in 187 msec 2024-11-28T10:41:16,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-28T10:41:16,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=e7e15bd2cf2816daa3a7922c2d73578c, ASSIGN in 355 msec 2024-11-28T10:41:16,966 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:16,966 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790476966"}]},"ts":"1732790476966"} 2024-11-28T10:41:16,969 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-28T10:41:16,971 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:16,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2120 sec 2024-11-28T10:41:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-28T10:41:18,794 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-28T10:41:18,794 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-28T10:41:18,794 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:18,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-28T10:41:18,800 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:18,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-28T10:41:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:18,816 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:18,817 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-28T10:41:18,820 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T10:41:18,823 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T10:41:18,824 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T10:41:18,825 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-28T10:41:20,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-28T10:41:20,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:22,463 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:41:22,696 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-28T10:41:24,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34367 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549227' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T10:41:24,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34367 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:41288 deadline: 1732790490827, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549227' does not exist 2024-11-28T10:41:24,833 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to e792e7e95410,34367,1732790431492 for closing scanner id = -3222361654058549227 for e7e15bd2cf2816daa3a7922c2d73578c of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549227' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T10:41:24,838 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=157 (was 168), ProcessCount=11 (was 11), AvailableMemoryMB=4650 (was 4678) 2024-11-28T10:41:24,847 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=157, ProcessCount=11, AvailableMemoryMB=4649 2024-11-28T10:41:24,849 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-28T10:41:24,852 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:24,853 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:24,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-28T10:41:24,854 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T10:41:24,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741841_1017 (size=423) 2024-11-28T10:41:25,266 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23815ef8124e6f55115a4c3d96238fa9, NAME => 'testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:25,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741842_1018 (size=74) 2024-11-28T10:41:25,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-28T10:41:25,677 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:25,678 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 23815ef8124e6f55115a4c3d96238fa9, disabling compactions & flushes 2024-11-28T10:41:25,678 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:41:25,678 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:41:25,678 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. after waiting 0 ms 2024-11-28T10:41:25,678 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:41:25,678 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:41:25,678 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23815ef8124e6f55115a4c3d96238fa9: Waiting for close lock at 1732790485678Disabling compacts and flushes for region at 1732790485678Disabling writes for close at 1732790485678Writing region close event to WAL at 1732790485678Closed at 1732790485678 2024-11-28T10:41:25,681 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:25,681 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732790485681"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790485681"}]},"ts":"1732790485681"} 2024-11-28T10:41:25,686 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:25,688 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:25,688 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790485688"}]},"ts":"1732790485688"} 2024-11-28T10:41:25,691 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-28T10:41:25,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=23815ef8124e6f55115a4c3d96238fa9, ASSIGN}] 2024-11-28T10:41:25,693 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=23815ef8124e6f55115a4c3d96238fa9, ASSIGN 2024-11-28T10:41:25,695 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=23815ef8124e6f55115a4c3d96238fa9, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:25,846 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=23815ef8124e6f55115a4c3d96238fa9, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:25,851 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=23815ef8124e6f55115a4c3d96238fa9, ASSIGN because future has completed 2024-11-28T10:41:25,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23815ef8124e6f55115a4c3d96238fa9, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T10:41:26,015 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:41:26,015 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 23815ef8124e6f55115a4c3d96238fa9, NAME => 'testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:26,015 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,016 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:26,016 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,016 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,020 INFO [StoreOpener-23815ef8124e6f55115a4c3d96238fa9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,023 INFO [StoreOpener-23815ef8124e6f55115a4c3d96238fa9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23815ef8124e6f55115a4c3d96238fa9 columnFamilyName testFamily 2024-11-28T10:41:26,023 DEBUG [StoreOpener-23815ef8124e6f55115a4c3d96238fa9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:26,024 INFO [StoreOpener-23815ef8124e6f55115a4c3d96238fa9-1 {}] regionserver.HStore(327): Store=23815ef8124e6f55115a4c3d96238fa9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:26,024 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,026 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,026 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,027 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,027 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,030 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,037 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:26,038 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 23815ef8124e6f55115a4c3d96238fa9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66332289, jitterRate=-0.011571869254112244}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:26,038 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:41:26,038 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 23815ef8124e6f55115a4c3d96238fa9: Running coprocessor pre-open hook at 1732790486016Writing region info on filesystem at 1732790486016Initializing all the Stores at 1732790486018 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790486018Cleaning up temporary data from old regions at 1732790486027 (+9 ms)Running coprocessor post-open hooks at 1732790486038 (+11 ms)Region opened successfully at 1732790486038 2024-11-28T10:41:26,040 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9., pid=15, masterSystemTime=1732790486007 2024-11-28T10:41:26,042 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:41:26,042 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:41:26,043 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=23815ef8124e6f55115a4c3d96238fa9, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:26,046 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23815ef8124e6f55115a4c3d96238fa9, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:26,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-28T10:41:26,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 23815ef8124e6f55115a4c3d96238fa9, server=e792e7e95410,34367,1732790431492 in 196 msec 2024-11-28T10:41:26,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-28T10:41:26,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=23815ef8124e6f55115a4c3d96238fa9, ASSIGN in 359 msec 2024-11-28T10:41:26,056 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:26,056 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790486056"}]},"ts":"1732790486056"} 2024-11-28T10:41:26,059 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-28T10:41:26,060 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:26,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2100 sec 2024-11-28T10:41:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T10:41:27,873 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-28T10:41:27,874 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-28T10:41:27,874 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:27,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-28T10:41:27,878 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:27,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-28T10:41:27,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:27,895 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:27,895 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-28T10:41:27,899 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-28T10:41:27,902 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-28T10:41:27,904 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-28T10:41:27,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3222361654058549223 2024-11-28T10:41:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41288 deadline: 1732790490903, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T10:41:27,909 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-28T10:41:27,910 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-28T10:41:27,911 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:27,912 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:27,913 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3222361654058549221 2024-11-28T10:41:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41288 deadline: 1732790490912, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T10:41:27,916 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:27,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3222361654058549220 2024-11-28T10:41:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41288 deadline: 1732790490915, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T10:41:27,920 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:27,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3222361654058549219 2024-11-28T10:41:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:41288 deadline: 1732790490919, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T10:41:27,923 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T10:41:27,931 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_891951882_22 at /127.0.0.1:51140 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=157 (was 157), ProcessCount=11 (was 11), AvailableMemoryMB=4630 (was 4649) 2024-11-28T10:41:27,939 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=157, ProcessCount=11, AvailableMemoryMB=4630 2024-11-28T10:41:27,942 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-28T10:41:27,946 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:27,946 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:27,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-28T10:41:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T10:41:27,947 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:27,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741843_1019 (size=418) 2024-11-28T10:41:28,363 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d82f136f54abbcb5c692193a8459d46d, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:28,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741844_1020 (size=69) 2024-11-28T10:41:28,700 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-28T10:41:28,778 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:28,778 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing d82f136f54abbcb5c692193a8459d46d, disabling compactions & flushes 2024-11-28T10:41:28,778 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:41:28,779 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:41:28,779 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. after waiting 0 ms 2024-11-28T10:41:28,779 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:41:28,779 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:41:28,779 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for d82f136f54abbcb5c692193a8459d46d: Waiting for close lock at 1732790488778Disabling compacts and flushes for region at 1732790488778Disabling writes for close at 1732790488779 (+1 ms)Writing region close event to WAL at 1732790488779Closed at 1732790488779 2024-11-28T10:41:28,784 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:28,785 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732790488784"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790488784"}]},"ts":"1732790488784"} 2024-11-28T10:41:28,789 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:28,791 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:28,791 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790488791"}]},"ts":"1732790488791"} 2024-11-28T10:41:28,795 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-28T10:41:28,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d82f136f54abbcb5c692193a8459d46d, ASSIGN}] 2024-11-28T10:41:28,798 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d82f136f54abbcb5c692193a8459d46d, ASSIGN 2024-11-28T10:41:28,799 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d82f136f54abbcb5c692193a8459d46d, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:28,951 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d82f136f54abbcb5c692193a8459d46d, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T10:41:28,957 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d82f136f54abbcb5c692193a8459d46d, ASSIGN because future has completed 2024-11-28T10:41:28,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d82f136f54abbcb5c692193a8459d46d, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:29,123 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:41:29,123 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => d82f136f54abbcb5c692193a8459d46d, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:29,124 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,124 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:29,124 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,124 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,126 INFO [StoreOpener-d82f136f54abbcb5c692193a8459d46d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,128 INFO [StoreOpener-d82f136f54abbcb5c692193a8459d46d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d82f136f54abbcb5c692193a8459d46d columnFamilyName testFamily 2024-11-28T10:41:29,128 DEBUG [StoreOpener-d82f136f54abbcb5c692193a8459d46d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:29,128 INFO [StoreOpener-d82f136f54abbcb5c692193a8459d46d-1 {}] regionserver.HStore(327): Store=d82f136f54abbcb5c692193a8459d46d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:29,129 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,130 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,130 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,131 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,131 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,134 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,137 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:29,137 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened d82f136f54abbcb5c692193a8459d46d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61997422, jitterRate=-0.07616642117500305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:29,138 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:41:29,139 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for d82f136f54abbcb5c692193a8459d46d: Running coprocessor pre-open hook at 1732790489124Writing region info on filesystem at 1732790489124Initializing all the Stores at 1732790489125 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790489126 (+1 ms)Cleaning up temporary data from old regions at 1732790489131 (+5 ms)Running coprocessor post-open hooks at 1732790489138 (+7 ms)Region opened successfully at 1732790489138 2024-11-28T10:41:29,140 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., pid=18, masterSystemTime=1732790489112 2024-11-28T10:41:29,143 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:41:29,143 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:41:29,144 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=d82f136f54abbcb5c692193a8459d46d, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:29,146 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure d82f136f54abbcb5c692193a8459d46d, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:29,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T10:41:29,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure d82f136f54abbcb5c692193a8459d46d, server=e792e7e95410,34367,1732790431492 in 191 msec 2024-11-28T10:41:29,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-28T10:41:29,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=d82f136f54abbcb5c692193a8459d46d, ASSIGN in 356 msec 2024-11-28T10:41:29,157 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:29,158 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790489158"}]},"ts":"1732790489158"} 2024-11-28T10:41:29,161 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-28T10:41:29,163 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:29,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2210 sec 2024-11-28T10:41:29,523 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T10:41:30,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-28T10:41:30,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:30,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-28T10:41:30,715 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T10:41:30,976 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-28T10:41:30,976 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-28T10:41:30,977 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:30,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-28T10:41:30,983 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:30,983 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-28T10:41:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:30,999 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:30,999 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-28T10:41:31,002 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T10:41:31,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T10:41:31,518 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:41:32,005 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e792e7e95410:34367 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-11-28T10:41:32,005 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-11-28T10:41:32,005 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2 from cache 2024-11-28T10:41:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-28T10:41:33,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:33,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T10:41:34,004 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e792e7e95410:34367 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=982ms, rpcTimeout=980ms 2024-11-28T10:41:34,005 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=982ms, rpcTimeout=980ms 2024-11-28T10:41:34,005 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., hostname=e792e7e95410,34367,1732790431492, seqNum=2 from cache 2024-11-28T10:41:34,018 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=145 (was 157), ProcessCount=11 (was 11), AvailableMemoryMB=4619 (was 4630) 2024-11-28T10:41:34,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=4619 2024-11-28T10:41:34,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-28T10:41:34,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:34,039 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:34,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-28T10:41:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T10:41:34,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:34,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741845_1021 (size=416) 2024-11-28T10:41:34,458 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bc5ffa67250885deceb033d7389a32ed, NAME => 'testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:34,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741846_1022 (size=67) 2024-11-28T10:41:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-28T10:41:34,703 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-28T10:41:34,871 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:34,871 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing bc5ffa67250885deceb033d7389a32ed, disabling compactions & flushes 2024-11-28T10:41:34,871 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:41:34,872 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:41:34,872 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. after waiting 0 ms 2024-11-28T10:41:34,872 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:41:34,872 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:41:34,873 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for bc5ffa67250885deceb033d7389a32ed: Waiting for close lock at 1732790494871Disabling compacts and flushes for region at 1732790494871Disabling writes for close at 1732790494872 (+1 ms)Writing region close event to WAL at 1732790494872Closed at 1732790494872 2024-11-28T10:41:34,913 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:34,913 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732790494913"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790494913"}]},"ts":"1732790494913"} 2024-11-28T10:41:34,916 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:34,918 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:34,918 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790494918"}]},"ts":"1732790494918"} 2024-11-28T10:41:34,921 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-28T10:41:34,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bc5ffa67250885deceb033d7389a32ed, ASSIGN}] 2024-11-28T10:41:34,924 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bc5ffa67250885deceb033d7389a32ed, ASSIGN 2024-11-28T10:41:34,925 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bc5ffa67250885deceb033d7389a32ed, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T10:41:35,077 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=bc5ffa67250885deceb033d7389a32ed, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:35,084 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bc5ffa67250885deceb033d7389a32ed, ASSIGN because future has completed 2024-11-28T10:41:35,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc5ffa67250885deceb033d7389a32ed, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:35,248 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:41:35,248 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => bc5ffa67250885deceb033d7389a32ed, NAME => 'testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:35,249 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,249 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:35,249 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,249 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,251 INFO [StoreOpener-bc5ffa67250885deceb033d7389a32ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,253 INFO [StoreOpener-bc5ffa67250885deceb033d7389a32ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc5ffa67250885deceb033d7389a32ed columnFamilyName testFamily 2024-11-28T10:41:35,253 DEBUG [StoreOpener-bc5ffa67250885deceb033d7389a32ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:35,254 INFO [StoreOpener-bc5ffa67250885deceb033d7389a32ed-1 {}] regionserver.HStore(327): Store=bc5ffa67250885deceb033d7389a32ed/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:35,254 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,255 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,256 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,256 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,256 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,258 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,261 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:35,262 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened bc5ffa67250885deceb033d7389a32ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74684883, jitterRate=0.11289148032665253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:35,262 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:41:35,263 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for bc5ffa67250885deceb033d7389a32ed: Running coprocessor pre-open hook at 1732790495249Writing region info on filesystem at 1732790495249Initializing all the Stores at 1732790495251 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790495251Cleaning up temporary data from old regions at 1732790495256 (+5 ms)Running coprocessor post-open hooks at 1732790495262 (+6 ms)Region opened successfully at 1732790495262 2024-11-28T10:41:35,264 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed., pid=21, masterSystemTime=1732790495237 2024-11-28T10:41:35,266 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:41:35,266 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:41:35,267 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=bc5ffa67250885deceb033d7389a32ed, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:35,270 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc5ffa67250885deceb033d7389a32ed, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:35,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-28T10:41:35,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure bc5ffa67250885deceb033d7389a32ed, server=e792e7e95410,34367,1732790431492 in 187 msec 2024-11-28T10:41:35,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-28T10:41:35,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=bc5ffa67250885deceb033d7389a32ed, ASSIGN in 352 msec 2024-11-28T10:41:35,278 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:35,278 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790495278"}]},"ts":"1732790495278"} 2024-11-28T10:41:35,280 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-28T10:41:35,281 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:35,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2480 sec 2024-11-28T10:41:37,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T10:41:37,065 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-28T10:41:37,065 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-28T10:41:37,066 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:37,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-28T10:41:37,074 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:37,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-28T10:41:37,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:37,088 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:37,089 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-28T10:41:37,091 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T10:41:37,094 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T10:41:37,095 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T10:41:37,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-28T10:41:40,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34367 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549208' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T10:41:40,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34367 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:41288 deadline: 1732790501098, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549208' does not exist 2024-11-28T10:41:40,099 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to e792e7e95410,34367,1732790431492 for closing scanner id = -3222361654058549208 for bc5ffa67250885deceb033d7389a32ed of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549208' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T10:41:40,105 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 145), ProcessCount=11 (was 11), AvailableMemoryMB=4616 (was 4619) 2024-11-28T10:41:40,116 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=4615 2024-11-28T10:41:40,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-28T10:41:40,123 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:40,123 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:40,124 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-28T10:41:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T10:41:40,125 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:40,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741847_1023 (size=411) 2024-11-28T10:41:40,541 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 61054584bf641bf46d8f877b534d70c1, NAME => 'testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:40,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741848_1024 (size=62) 2024-11-28T10:41:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-28T10:41:40,754 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:41:40,781 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-28T10:41:40,781 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-28T10:41:40,781 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:40,956 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:40,956 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 61054584bf641bf46d8f877b534d70c1, disabling compactions & flushes 2024-11-28T10:41:40,956 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:41:40,956 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:41:40,957 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. after waiting 0 ms 2024-11-28T10:41:40,957 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:41:40,957 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:41:40,957 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 61054584bf641bf46d8f877b534d70c1: Waiting for close lock at 1732790500956Disabling compacts and flushes for region at 1732790500956Disabling writes for close at 1732790500957 (+1 ms)Writing region close event to WAL at 1732790500957Closed at 1732790500957 2024-11-28T10:41:40,959 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:40,960 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732790500960"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790500960"}]},"ts":"1732790500960"} 2024-11-28T10:41:40,963 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:40,965 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:40,966 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790500965"}]},"ts":"1732790500965"} 2024-11-28T10:41:40,969 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-28T10:41:40,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=61054584bf641bf46d8f877b534d70c1, ASSIGN}] 2024-11-28T10:41:40,972 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=61054584bf641bf46d8f877b534d70c1, ASSIGN 2024-11-28T10:41:40,973 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=61054584bf641bf46d8f877b534d70c1, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:41,124 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=61054584bf641bf46d8f877b534d70c1, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:41,130 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=61054584bf641bf46d8f877b534d70c1, ASSIGN because future has completed 2024-11-28T10:41:41,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61054584bf641bf46d8f877b534d70c1, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T10:41:41,300 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:41:41,300 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 61054584bf641bf46d8f877b534d70c1, NAME => 'testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:41,301 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,301 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:41,301 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,301 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,303 INFO [StoreOpener-61054584bf641bf46d8f877b534d70c1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,305 INFO [StoreOpener-61054584bf641bf46d8f877b534d70c1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 61054584bf641bf46d8f877b534d70c1 columnFamilyName testFamily 2024-11-28T10:41:41,305 DEBUG [StoreOpener-61054584bf641bf46d8f877b534d70c1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:41,306 INFO [StoreOpener-61054584bf641bf46d8f877b534d70c1-1 {}] regionserver.HStore(327): Store=61054584bf641bf46d8f877b534d70c1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:41,306 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,307 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,308 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,309 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,309 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,311 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,315 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:41,315 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 61054584bf641bf46d8f877b534d70c1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67885654, jitterRate=0.011575073003768921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:41,316 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:41:41,317 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 61054584bf641bf46d8f877b534d70c1: Running coprocessor pre-open hook at 1732790501301Writing region info on filesystem at 1732790501301Initializing all the Stores at 1732790501303 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790501303Cleaning up temporary data from old regions at 1732790501309 (+6 ms)Running coprocessor post-open hooks at 1732790501316 (+7 ms)Region opened successfully at 1732790501316 2024-11-28T10:41:41,318 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1., pid=24, masterSystemTime=1732790501287 2024-11-28T10:41:41,320 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:41:41,320 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:41:41,321 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=61054584bf641bf46d8f877b534d70c1, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:41,324 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 61054584bf641bf46d8f877b534d70c1, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:41,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-28T10:41:41,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 61054584bf641bf46d8f877b534d70c1, server=e792e7e95410,34367,1732790431492 in 194 msec 2024-11-28T10:41:41,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-28T10:41:41,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=61054584bf641bf46d8f877b534d70c1, ASSIGN in 357 msec 2024-11-28T10:41:41,331 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:41,331 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790501331"}]},"ts":"1732790501331"} 2024-11-28T10:41:41,334 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-28T10:41:41,335 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:41,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2150 sec 2024-11-28T10:41:43,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T10:41:43,164 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-28T10:41:43,164 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-28T10:41:43,165 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:43,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-28T10:41:43,171 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:43,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-28T10:41:43,177 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:43,192 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:43,193 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-28T10:41:43,196 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T10:41:43,199 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T10:41:43,201 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T10:41:43,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-28T10:41:45,356 INFO [master/e792e7e95410:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T10:41:45,356 INFO [master/e792e7e95410:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T10:41:46,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34367 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549204' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T10:41:46,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34367 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:41288 deadline: 1732790507208, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549204' does not exist 2024-11-28T10:41:46,209 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to e792e7e95410,34367,1732790431492 for closing scanner id = -3222361654058549204 for 61054584bf641bf46d8f877b534d70c1 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549204' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T10:41:46,216 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=144 (was 130) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5267 (was 4615) - AvailableMemoryMB LEAK? - 2024-11-28T10:41:46,227 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=144, ProcessCount=11, AvailableMemoryMB=5266 2024-11-28T10:41:46,231 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:46,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-28T10:41:46,234 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:46,234 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:46,235 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-28T10:41:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T10:41:46,236 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:46,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741849_1025 (size=419) 2024-11-28T10:41:46,652 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3f337989cad19405d44b969f951fb910, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:46,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741850_1026 (size=70) 2024-11-28T10:41:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-28T10:41:46,803 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:41:46,832 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-28T10:41:47,061 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:47,061 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 3f337989cad19405d44b969f951fb910, disabling compactions & flushes 2024-11-28T10:41:47,061 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:41:47,062 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:41:47,062 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. after waiting 0 ms 2024-11-28T10:41:47,062 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:41:47,062 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:41:47,062 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3f337989cad19405d44b969f951fb910: Waiting for close lock at 1732790507061Disabling compacts and flushes for region at 1732790507061Disabling writes for close at 1732790507062 (+1 ms)Writing region close event to WAL at 1732790507062Closed at 1732790507062 2024-11-28T10:41:47,063 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:47,064 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732790507063"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790507063"}]},"ts":"1732790507063"} 2024-11-28T10:41:47,066 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:47,068 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:47,068 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790507068"}]},"ts":"1732790507068"} 2024-11-28T10:41:47,071 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-28T10:41:47,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3f337989cad19405d44b969f951fb910, ASSIGN}] 2024-11-28T10:41:47,073 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3f337989cad19405d44b969f951fb910, ASSIGN 2024-11-28T10:41:47,074 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3f337989cad19405d44b969f951fb910, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:47,225 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=3f337989cad19405d44b969f951fb910, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:47,228 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3f337989cad19405d44b969f951fb910, ASSIGN because future has completed 2024-11-28T10:41:47,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f337989cad19405d44b969f951fb910, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T10:41:47,386 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:41:47,386 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f337989cad19405d44b969f951fb910, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:47,386 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,386 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:47,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,389 INFO [StoreOpener-3f337989cad19405d44b969f951fb910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,391 INFO [StoreOpener-3f337989cad19405d44b969f951fb910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f337989cad19405d44b969f951fb910 columnFamilyName testFamily 2024-11-28T10:41:47,392 DEBUG [StoreOpener-3f337989cad19405d44b969f951fb910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:47,392 INFO [StoreOpener-3f337989cad19405d44b969f951fb910-1 {}] regionserver.HStore(327): Store=3f337989cad19405d44b969f951fb910/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:47,393 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,394 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,394 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,395 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,398 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,401 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:47,402 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 3f337989cad19405d44b969f951fb910; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75266915, jitterRate=0.12156443297863007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:47,402 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f337989cad19405d44b969f951fb910 2024-11-28T10:41:47,402 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 3f337989cad19405d44b969f951fb910: Running coprocessor pre-open hook at 1732790507387Writing region info on filesystem at 1732790507387Initializing all the Stores at 1732790507388 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790507388Cleaning up temporary data from old regions at 1732790507395 (+7 ms)Running coprocessor post-open hooks at 1732790507402 (+7 ms)Region opened successfully at 1732790507402 2024-11-28T10:41:47,404 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910., pid=27, masterSystemTime=1732790507380 2024-11-28T10:41:47,407 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:41:47,407 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:41:47,408 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=3f337989cad19405d44b969f951fb910, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:47,411 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f337989cad19405d44b969f951fb910, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:47,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-28T10:41:47,415 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f337989cad19405d44b969f951fb910, server=e792e7e95410,34367,1732790431492 in 184 msec 2024-11-28T10:41:47,418 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-28T10:41:47,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3f337989cad19405d44b969f951fb910, ASSIGN in 344 msec 2024-11-28T10:41:47,420 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:47,420 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790507420"}]},"ts":"1732790507420"} 2024-11-28T10:41:47,423 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-28T10:41:47,424 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:47,427 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1930 sec 2024-11-28T10:41:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T10:41:49,264 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-28T10:41:49,264 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-28T10:41:49,265 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:49,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-28T10:41:49,269 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:49,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-28T10:41:49,272 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:49,282 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:49,282 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-28T10:41:49,285 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T10:41:49,286 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-28T10:41:50,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-28T10:41:50,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:50,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-28T10:41:50,714 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T10:41:52,889 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:41:52,924 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-28T10:41:55,295 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910., hostname=e792e7e95410,34367,1732790431492, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910., hostname=e792e7e95410,34367,1732790431492, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e792e7e95410:34367 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-28T10:41:55,295 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910., hostname=e792e7e95410,34367,1732790431492, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-28T10:41:55,295 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910., hostname=e792e7e95410,34367,1732790431492, seqNum=2 from cache 2024-11-28T10:41:55,308 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 144) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4996 (was 5266) 2024-11-28T10:41:55,318 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=168, ProcessCount=11, AvailableMemoryMB=4995 2024-11-28T10:41:55,322 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:41:55,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-28T10:41:55,326 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:41:55,326 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:55,326 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-28T10:41:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-28T10:41:55,328 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:41:55,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741851_1027 (size=407) 2024-11-28T10:41:55,345 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94a73c4fa948020f43325f8b1c60ff43, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:41:55,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741852_1028 (size=58) 2024-11-28T10:41:55,355 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:55,355 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 94a73c4fa948020f43325f8b1c60ff43, disabling compactions & flushes 2024-11-28T10:41:55,356 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:41:55,356 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:41:55,356 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. after waiting 0 ms 2024-11-28T10:41:55,356 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:41:55,356 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:41:55,356 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94a73c4fa948020f43325f8b1c60ff43: Waiting for close lock at 1732790515355Disabling compacts and flushes for region at 1732790515355Disabling writes for close at 1732790515356 (+1 ms)Writing region close event to WAL at 1732790515356Closed at 1732790515356 2024-11-28T10:41:55,358 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:41:55,358 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732790515358"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790515358"}]},"ts":"1732790515358"} 2024-11-28T10:41:55,361 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:41:55,363 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:41:55,363 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790515363"}]},"ts":"1732790515363"} 2024-11-28T10:41:55,366 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-28T10:41:55,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=94a73c4fa948020f43325f8b1c60ff43, ASSIGN}] 2024-11-28T10:41:55,369 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=94a73c4fa948020f43325f8b1c60ff43, ASSIGN 2024-11-28T10:41:55,370 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=94a73c4fa948020f43325f8b1c60ff43, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:41:55,521 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=94a73c4fa948020f43325f8b1c60ff43, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:55,525 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=94a73c4fa948020f43325f8b1c60ff43, ASSIGN because future has completed 2024-11-28T10:41:55,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94a73c4fa948020f43325f8b1c60ff43, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:41:55,685 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:41:55,685 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 94a73c4fa948020f43325f8b1c60ff43, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:41:55,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:41:55,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,688 INFO [StoreOpener-94a73c4fa948020f43325f8b1c60ff43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,690 INFO [StoreOpener-94a73c4fa948020f43325f8b1c60ff43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94a73c4fa948020f43325f8b1c60ff43 columnFamilyName testFamily 2024-11-28T10:41:55,691 DEBUG [StoreOpener-94a73c4fa948020f43325f8b1c60ff43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:41:55,692 INFO [StoreOpener-94a73c4fa948020f43325f8b1c60ff43-1 {}] regionserver.HStore(327): Store=94a73c4fa948020f43325f8b1c60ff43/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:41:55,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,693 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,694 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,695 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,695 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,697 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,700 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:41:55,702 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 94a73c4fa948020f43325f8b1c60ff43; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65563133, jitterRate=-0.02303318679332733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:41:55,702 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:41:55,703 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 94a73c4fa948020f43325f8b1c60ff43: Running coprocessor pre-open hook at 1732790515686Writing region info on filesystem at 1732790515686Initializing all the Stores at 1732790515687 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790515688 (+1 ms)Cleaning up temporary data from old regions at 1732790515695 (+7 ms)Running coprocessor post-open hooks at 1732790515702 (+7 ms)Region opened successfully at 1732790515703 (+1 ms) 2024-11-28T10:41:55,705 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43., pid=30, masterSystemTime=1732790515679 2024-11-28T10:41:55,708 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:41:55,708 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:41:55,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=94a73c4fa948020f43325f8b1c60ff43, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:41:55,713 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94a73c4fa948020f43325f8b1c60ff43, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:41:55,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-28T10:41:55,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 94a73c4fa948020f43325f8b1c60ff43, server=e792e7e95410,34367,1732790431492 in 193 msec 2024-11-28T10:41:55,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-28T10:41:55,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=94a73c4fa948020f43325f8b1c60ff43, ASSIGN in 356 msec 2024-11-28T10:41:55,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:41:55,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790515729"}]},"ts":"1732790515729"} 2024-11-28T10:41:55,733 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-28T10:41:55,735 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:41:55,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 413 msec 2024-11-28T10:41:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-28T10:41:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-28T10:41:56,333 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-28T10:41:56,333 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-28T10:41:56,334 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:56,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-28T10:41:56,340 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:41:56,341 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-28T10:41:56,344 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:41:56,359 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:41:56,359 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-28T10:41:56,363 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T10:41:56,367 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T10:41:56,368 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T10:41:56,369 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-28T10:41:58,925 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-28T10:41:59,524 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T10:42:01,190 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:42:01,220 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-28T10:42:01,220 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-28T10:42:02,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34367 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549196' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T10:42:02,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34367 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:41288 deadline: 1732790528375, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549196' does not exist 2024-11-28T10:42:02,379 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to e792e7e95410,34367,1732790431492 for closing scanner id = -3222361654058549196 for 94a73c4fa948020f43325f8b1c60ff43 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3222361654058549196' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T10:42:02,385 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=155 (was 168), ProcessCount=11 (was 11), AvailableMemoryMB=4889 (was 4995) 2024-11-28T10:42:02,393 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=155, ProcessCount=11, AvailableMemoryMB=4889 2024-11-28T10:42:02,395 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T10:42:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-28T10:42:02,397 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T10:42:02,397 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:42:02,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-28T10:42:02,398 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T10:42:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T10:42:02,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741853_1029 (size=423) 2024-11-28T10:42:02,813 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2ec26e43dfdf13098af7fe3044ccf73c, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226 2024-11-28T10:42:02,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741854_1030 (size=74) 2024-11-28T10:42:02,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-28T10:42:03,227 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:42:03,227 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 2ec26e43dfdf13098af7fe3044ccf73c, disabling compactions & flushes 2024-11-28T10:42:03,227 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:03,227 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:03,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. after waiting 0 ms 2024-11-28T10:42:03,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:03,228 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:03,228 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2ec26e43dfdf13098af7fe3044ccf73c: Waiting for close lock at 1732790523227Disabling compacts and flushes for region at 1732790523227Disabling writes for close at 1732790523228 (+1 ms)Writing region close event to WAL at 1732790523228Closed at 1732790523228 2024-11-28T10:42:03,231 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T10:42:03,232 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732790523231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732790523231"}]},"ts":"1732790523231"} 2024-11-28T10:42:03,236 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T10:42:03,238 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T10:42:03,238 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790523238"}]},"ts":"1732790523238"} 2024-11-28T10:42:03,241 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-28T10:42:03,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2ec26e43dfdf13098af7fe3044ccf73c, ASSIGN}] 2024-11-28T10:42:03,243 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2ec26e43dfdf13098af7fe3044ccf73c, ASSIGN 2024-11-28T10:42:03,244 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2ec26e43dfdf13098af7fe3044ccf73c, ASSIGN; state=OFFLINE, location=e792e7e95410,34367,1732790431492; forceNewPlan=false, retain=false 2024-11-28T10:42:03,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=2ec26e43dfdf13098af7fe3044ccf73c, regionState=OPENING, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:42:03,400 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2ec26e43dfdf13098af7fe3044ccf73c, ASSIGN because future has completed 2024-11-28T10:42:03,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ec26e43dfdf13098af7fe3044ccf73c, server=e792e7e95410,34367,1732790431492}] 2024-11-28T10:42:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T10:42:03,564 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:03,564 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ec26e43dfdf13098af7fe3044ccf73c, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c.', STARTKEY => '', ENDKEY => ''} 2024-11-28T10:42:03,565 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,565 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T10:42:03,565 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,565 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,568 INFO [StoreOpener-2ec26e43dfdf13098af7fe3044ccf73c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,571 INFO [StoreOpener-2ec26e43dfdf13098af7fe3044ccf73c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ec26e43dfdf13098af7fe3044ccf73c columnFamilyName testFamily 2024-11-28T10:42:03,571 DEBUG [StoreOpener-2ec26e43dfdf13098af7fe3044ccf73c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T10:42:03,572 INFO [StoreOpener-2ec26e43dfdf13098af7fe3044ccf73c-1 {}] regionserver.HStore(327): Store=2ec26e43dfdf13098af7fe3044ccf73c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T10:42:03,572 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,574 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,574 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,575 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,575 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,578 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,582 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T10:42:03,583 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 2ec26e43dfdf13098af7fe3044ccf73c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60283375, jitterRate=-0.10170771181583405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T10:42:03,583 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:03,584 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 2ec26e43dfdf13098af7fe3044ccf73c: Running coprocessor pre-open hook at 1732790523566Writing region info on filesystem at 1732790523566Initializing all the Stores at 1732790523567 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732790523567Cleaning up temporary data from old regions at 1732790523575 (+8 ms)Running coprocessor post-open hooks at 1732790523583 (+8 ms)Region opened successfully at 1732790523584 (+1 ms) 2024-11-28T10:42:03,585 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., pid=33, masterSystemTime=1732790523554 2024-11-28T10:42:03,588 DEBUG [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:03,588 INFO [RS_OPEN_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:03,589 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=2ec26e43dfdf13098af7fe3044ccf73c, regionState=OPEN, openSeqNum=2, regionLocation=e792e7e95410,34367,1732790431492 2024-11-28T10:42:03,591 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ec26e43dfdf13098af7fe3044ccf73c, server=e792e7e95410,34367,1732790431492 because future has completed 2024-11-28T10:42:03,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-28T10:42:03,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ec26e43dfdf13098af7fe3044ccf73c, server=e792e7e95410,34367,1732790431492 in 193 msec 2024-11-28T10:42:03,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-28T10:42:03,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=2ec26e43dfdf13098af7fe3044ccf73c, ASSIGN in 355 msec 2024-11-28T10:42:03,600 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T10:42:03,600 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732790523600"}]},"ts":"1732790523600"} 2024-11-28T10:42:03,603 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-28T10:42:03,604 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T10:42:03,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2090 sec 2024-11-28T10:42:04,928 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-28T10:42:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T10:42:05,435 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-28T10:42:05,435 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-28T10:42:05,436 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:42:05,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-28T10:42:05,447 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T10:42:05,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-28T10:42:05,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:42:05,464 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T10:42:05,464 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-28T10:42:05,468 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T10:42:05,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T10:42:05,574 DEBUG [master/e792e7e95410:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-28T10:42:06,475 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e792e7e95410:34367 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-28T10:42:06,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-28T10:42:06,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2 from cache 2024-11-28T10:42:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-28T10:42:07,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2] 2024-11-28T10:42:07,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34367 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T10:42:08,475 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=e792e7e95410:34367 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=987ms, rpcTimeout=981ms 2024-11-28T10:42:08,475 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=987ms, rpcTimeout=981ms 2024-11-28T10:42:08,475 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., hostname=e792e7e95410,34367,1732790431492, seqNum=2 from cache 2024-11-28T10:42:08,488 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 155), ProcessCount=11 (was 11), AvailableMemoryMB=4853 (was 4889) 2024-11-28T10:42:08,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T10:42:08,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T10:42:08,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:42:08,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:42:08,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T10:42:08,495 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T10:42:08,495 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T10:42:08,496 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T10:42:08,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:42:08,496 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:42:08,496 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T10:42:08,496 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T10:42:08,496 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2107674261, stopped=false 2024-11-28T10:42:08,496 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e792e7e95410,42151,1732790430369 2024-11-28T10:42:08,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T10:42:08,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T10:42:08,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:42:08,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:42:08,626 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T10:42:08,627 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T10:42:08,627 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T10:42:08,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T10:42:08,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T10:42:08,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:42:08,628 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e792e7e95410,34367,1732790431492' ***** 2024-11-28T10:42:08,629 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T10:42:08,629 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T10:42:08,630 INFO [RS:0;e792e7e95410:34367 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T10:42:08,630 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T10:42:08,630 INFO [RS:0;e792e7e95410:34367 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T10:42:08,630 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for 3f337989cad19405d44b969f951fb910 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for d82f136f54abbcb5c692193a8459d46d 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for e7b909729cbd1c752cd16aaf58c8e110 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for 23815ef8124e6f55115a4c3d96238fa9 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for 94a73c4fa948020f43325f8b1c60ff43 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for 2ec26e43dfdf13098af7fe3044ccf73c 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:42:08,631 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3f337989cad19405d44b969f951fb910, disabling compactions & flushes 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:42:08,631 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:42:08,631 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:42:08,631 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:42:08,632 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(959): stopping server e792e7e95410,34367,1732790431492 2024-11-28T10:42:08,632 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. after waiting 0 ms 2024-11-28T10:42:08,632 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:42:08,632 INFO [RS:0;e792e7e95410:34367 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T10:42:08,632 INFO [RS:0;e792e7e95410:34367 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e792e7e95410:34367. 2024-11-28T10:42:08,632 DEBUG [RS:0;e792e7e95410:34367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T10:42:08,632 DEBUG [RS:0;e792e7e95410:34367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:42:08,632 INFO [RS:0;e792e7e95410:34367 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T10:42:08,632 INFO [RS:0;e792e7e95410:34367 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T10:42:08,632 INFO [RS:0;e792e7e95410:34367 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T10:42:08,632 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T10:42:08,633 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-28T10:42:08,633 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T10:42:08,633 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T10:42:08,633 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T10:42:08,633 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T10:42:08,633 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T10:42:08,633 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23815ef8124e6f55115a4c3d96238fa9, 2ec26e43dfdf13098af7fe3044ccf73c, 3f337989cad19405d44b969f951fb910, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, d82f136f54abbcb5c692193a8459d46d, e7b909729cbd1c752cd16aaf58c8e110, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:08,636 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3f337989cad19405d44b969f951fb910 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:08,636 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-28T10:42:08,685 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910/.tmp/testFamily/55b68708190a4bcd88e1bd8a1179917b is 61, key is row-0/testFamily:testQualifier/1732790509273/Put/seqid=0 2024-11-28T10:42:08,692 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/info/69fc212864354cb093d8de0a99d5d788 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6./info:regioninfo/1732790466323/Put/seqid=0 2024-11-28T10:42:08,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741855_1031 (size=5049) 2024-11-28T10:42:08,696 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910/.tmp/testFamily/55b68708190a4bcd88e1bd8a1179917b 2024-11-28T10:42:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741856_1032 (size=20564) 2024-11-28T10:42:08,698 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/info/69fc212864354cb093d8de0a99d5d788 2024-11-28T10:42:08,740 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910/.tmp/testFamily/55b68708190a4bcd88e1bd8a1179917b as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910/testFamily/55b68708190a4bcd88e1bd8a1179917b 2024-11-28T10:42:08,750 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910/testFamily/55b68708190a4bcd88e1bd8a1179917b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:08,755 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/ns/96c165c1542c4b3996c1136cdc489335 is 43, key is default/ns:d/1732790465260/Put/seqid=0 2024-11-28T10:42:08,758 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3f337989cad19405d44b969f951fb910 in 122ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:08,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741857_1033 (size=5153) 2024-11-28T10:42:08,764 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3f337989cad19405d44b969f951fb910/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:08,766 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:42:08,766 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3f337989cad19405d44b969f951fb910: Waiting for close lock at 1732790528631Running coprocessor pre-close hooks at 1732790528631Disabling compacts and flushes for region at 1732790528631Disabling writes for close at 1732790528632 (+1 ms)Obtaining lock to block concurrent updates at 1732790528636 (+4 ms)Preparing flush snapshotting stores in 3f337989cad19405d44b969f951fb910 at 1732790528636Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790528643 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. at 1732790528644 (+1 ms)Flushing 3f337989cad19405d44b969f951fb910/testFamily: creating writer at 1732790528646 (+2 ms)Flushing 3f337989cad19405d44b969f951fb910/testFamily: appending metadata at 1732790528680 (+34 ms)Flushing 3f337989cad19405d44b969f951fb910/testFamily: closing flushed file at 1732790528683 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64a7f09d: reopening flushed file at 1732790528738 (+55 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3f337989cad19405d44b969f951fb910 in 122ms, sequenceid=8, compaction requested=false at 1732790528758 (+20 ms)Writing region close event to WAL at 1732790528760 (+2 ms)Running coprocessor post-close hooks at 1732790528765 (+5 ms)Closed at 1732790528766 (+1 ms) 2024-11-28T10:42:08,767 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732790506230.3f337989cad19405d44b969f951fb910. 2024-11-28T10:42:08,767 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d82f136f54abbcb5c692193a8459d46d, disabling compactions & flushes 2024-11-28T10:42:08,767 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:42:08,767 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:42:08,767 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. after waiting 0 ms 2024-11-28T10:42:08,767 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:42:08,767 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d82f136f54abbcb5c692193a8459d46d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:08,784 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d/.tmp/testFamily/76a9bb67ef0e486ba62b22f296c9bf77 is 61, key is row-0/testFamily:testQualifier/1732790490989/Put/seqid=0 2024-11-28T10:42:08,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741858_1034 (size=5049) 2024-11-28T10:42:08,834 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23815ef8124e6f55115a4c3d96238fa9, 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, d82f136f54abbcb5c692193a8459d46d, e7b909729cbd1c752cd16aaf58c8e110, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:08,990 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-28T10:42:09,034 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23815ef8124e6f55115a4c3d96238fa9, 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, d82f136f54abbcb5c692193a8459d46d, e7b909729cbd1c752cd16aaf58c8e110, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:09,068 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:42:09,162 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/ns/96c165c1542c4b3996c1136cdc489335 2024-11-28T10:42:09,188 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/table/f14cf949816a40d68a872317f71400a9 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732790466337/Put/seqid=0 2024-11-28T10:42:09,190 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d/.tmp/testFamily/76a9bb67ef0e486ba62b22f296c9bf77 2024-11-28T10:42:09,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741859_1035 (size=6699) 2024-11-28T10:42:09,200 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d/.tmp/testFamily/76a9bb67ef0e486ba62b22f296c9bf77 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d/testFamily/76a9bb67ef0e486ba62b22f296c9bf77 2024-11-28T10:42:09,209 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d/testFamily/76a9bb67ef0e486ba62b22f296c9bf77, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:09,210 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d82f136f54abbcb5c692193a8459d46d in 443ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:09,215 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScanner/d82f136f54abbcb5c692193a8459d46d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:09,216 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:42:09,216 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d82f136f54abbcb5c692193a8459d46d: Waiting for close lock at 1732790528767Running coprocessor pre-close hooks at 1732790528767Disabling compacts and flushes for region at 1732790528767Disabling writes for close at 1732790528767Obtaining lock to block concurrent updates at 1732790528767Preparing flush snapshotting stores in d82f136f54abbcb5c692193a8459d46d at 1732790528767Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790528768 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. at 1732790528768Flushing d82f136f54abbcb5c692193a8459d46d/testFamily: creating writer at 1732790528769 (+1 ms)Flushing d82f136f54abbcb5c692193a8459d46d/testFamily: appending metadata at 1732790528784 (+15 ms)Flushing d82f136f54abbcb5c692193a8459d46d/testFamily: closing flushed file at 1732790528784Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@337cd5dd: reopening flushed file at 1732790529199 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d82f136f54abbcb5c692193a8459d46d in 443ms, sequenceid=8, compaction requested=false at 1732790529210 (+11 ms)Writing region close event to WAL at 1732790529211 (+1 ms)Running coprocessor post-close hooks at 1732790529216 (+5 ms)Closed at 1732790529216 2024-11-28T10:42:09,217 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732790487942.d82f136f54abbcb5c692193a8459d46d. 2024-11-28T10:42:09,217 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7b909729cbd1c752cd16aaf58c8e110, disabling compactions & flushes 2024-11-28T10:42:09,217 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:42:09,217 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:42:09,217 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. after waiting 0 ms 2024-11-28T10:42:09,217 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:42:09,217 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e7b909729cbd1c752cd16aaf58c8e110 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:09,235 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23815ef8124e6f55115a4c3d96238fa9, 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, e7b909729cbd1c752cd16aaf58c8e110, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:09,237 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110/.tmp/testFamily/3def1ca6b51b416a816b839f8e9a7fb6 is 61, key is row-0/testFamily:testQualifier/1732790469712/Put/seqid=0 2024-11-28T10:42:09,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741860_1036 (size=5049) 2024-11-28T10:42:09,435 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23815ef8124e6f55115a4c3d96238fa9, 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, e7b909729cbd1c752cd16aaf58c8e110, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:09,596 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/table/f14cf949816a40d68a872317f71400a9 2024-11-28T10:42:09,612 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/info/69fc212864354cb093d8de0a99d5d788 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/info/69fc212864354cb093d8de0a99d5d788 2024-11-28T10:42:09,621 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/info/69fc212864354cb093d8de0a99d5d788, entries=100, sequenceid=56, filesize=20.1 K 2024-11-28T10:42:09,622 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/ns/96c165c1542c4b3996c1136cdc489335 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/ns/96c165c1542c4b3996c1136cdc489335 2024-11-28T10:42:09,632 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/ns/96c165c1542c4b3996c1136cdc489335, entries=2, sequenceid=56, filesize=5.0 K 2024-11-28T10:42:09,633 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/.tmp/table/f14cf949816a40d68a872317f71400a9 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/table/f14cf949816a40d68a872317f71400a9 2024-11-28T10:42:09,635 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-28T10:42:09,635 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1325): Online Regions={e7b909729cbd1c752cd16aaf58c8e110=hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110., 23815ef8124e6f55115a4c3d96238fa9=testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9., 94a73c4fa948020f43325f8b1c60ff43=hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43., 1588230740=hbase:meta,,1.1588230740, 2ec26e43dfdf13098af7fe3044ccf73c=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., e99096e2f5934a127dfc49eb28ac61c6=testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6., e7e15bd2cf2816daa3a7922c2d73578c=hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c., bc5ffa67250885deceb033d7389a32ed=testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed., 61054584bf641bf46d8f877b534d70c1=testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.} 2024-11-28T10:42:09,635 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 23815ef8124e6f55115a4c3d96238fa9, 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, e7b909729cbd1c752cd16aaf58c8e110, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:09,642 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/table/f14cf949816a40d68a872317f71400a9, entries=20, sequenceid=56, filesize=6.5 K 2024-11-28T10:42:09,642 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110/.tmp/testFamily/3def1ca6b51b416a816b839f8e9a7fb6 2024-11-28T10:42:09,644 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1010ms, sequenceid=56, compaction requested=false 2024-11-28T10:42:09,648 INFO [regionserver/e792e7e95410:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T10:42:09,648 INFO [regionserver/e792e7e95410:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T10:42:09,649 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-28T10:42:09,651 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T10:42:09,651 INFO [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T10:42:09,651 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732790528633Running coprocessor pre-close hooks at 1732790528633Disabling compacts and flushes for region at 1732790528633Disabling writes for close at 1732790528633Obtaining lock to block concurrent updates at 1732790528636 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1732790528636Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732790528643 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732790528644 (+1 ms)Flushing 1588230740/info: creating writer at 1732790528646 (+2 ms)Flushing 1588230740/info: appending metadata at 1732790528686 (+40 ms)Flushing 1588230740/info: closing flushed file at 1732790528686Flushing 1588230740/ns: creating writer at 1732790528738 (+52 ms)Flushing 1588230740/ns: appending metadata at 1732790528754 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732790528754Flushing 1588230740/table: creating writer at 1732790529172 (+418 ms)Flushing 1588230740/table: appending metadata at 1732790529187 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732790529187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6db9ac5c: reopening flushed file at 1732790529610 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a4adff9: reopening flushed file at 1732790529621 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41809b4c: reopening flushed file at 1732790529632 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1010ms, sequenceid=56, compaction requested=false at 1732790529644 (+12 ms)Writing region close event to WAL at 1732790529645 (+1 ms)Running coprocessor post-close hooks at 1732790529651 (+6 ms)Closed at 1732790529651 2024-11-28T10:42:09,651 DEBUG [RS_CLOSE_META-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T10:42:09,652 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110/.tmp/testFamily/3def1ca6b51b416a816b839f8e9a7fb6 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110/testFamily/3def1ca6b51b416a816b839f8e9a7fb6 2024-11-28T10:42:09,660 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110/testFamily/3def1ca6b51b416a816b839f8e9a7fb6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:09,661 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e7b909729cbd1c752cd16aaf58c8e110 in 444ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:09,666 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnOpenScanner/e7b909729cbd1c752cd16aaf58c8e110/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:09,667 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:42:09,667 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7b909729cbd1c752cd16aaf58c8e110: Waiting for close lock at 1732790529217Running coprocessor pre-close hooks at 1732790529217Disabling compacts and flushes for region at 1732790529217Disabling writes for close at 1732790529217Obtaining lock to block concurrent updates at 1732790529217Preparing flush snapshotting stores in e7b909729cbd1c752cd16aaf58c8e110 at 1732790529217Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790529217Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. at 1732790529218 (+1 ms)Flushing e7b909729cbd1c752cd16aaf58c8e110/testFamily: creating writer at 1732790529218Flushing e7b909729cbd1c752cd16aaf58c8e110/testFamily: appending metadata at 1732790529236 (+18 ms)Flushing e7b909729cbd1c752cd16aaf58c8e110/testFamily: closing flushed file at 1732790529236Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5944e0bf: reopening flushed file at 1732790529651 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e7b909729cbd1c752cd16aaf58c8e110 in 444ms, sequenceid=8, compaction requested=false at 1732790529661 (+10 ms)Writing region close event to WAL at 1732790529661Running coprocessor post-close hooks at 1732790529667 (+6 ms)Closed at 1732790529667 2024-11-28T10:42:09,667 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732790466672.e7b909729cbd1c752cd16aaf58c8e110. 2024-11-28T10:42:09,667 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23815ef8124e6f55115a4c3d96238fa9, disabling compactions & flushes 2024-11-28T10:42:09,667 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:42:09,667 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:42:09,667 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. after waiting 0 ms 2024-11-28T10:42:09,667 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:42:09,667 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 23815ef8124e6f55115a4c3d96238fa9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:09,685 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9/.tmp/testFamily/d94dae741bb74aacae93008f9db5335e is 61, key is row-0/testFamily:testQualifier/1732790487883/Put/seqid=0 2024-11-28T10:42:09,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741861_1037 (size=5049) 2024-11-28T10:42:09,836 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 23815ef8124e6f55115a4c3d96238fa9, 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:10,036 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 23815ef8124e6f55115a4c3d96238fa9, 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:10,091 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9/.tmp/testFamily/d94dae741bb74aacae93008f9db5335e 2024-11-28T10:42:10,103 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9/.tmp/testFamily/d94dae741bb74aacae93008f9db5335e as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9/testFamily/d94dae741bb74aacae93008f9db5335e 2024-11-28T10:42:10,110 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9/testFamily/d94dae741bb74aacae93008f9db5335e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:10,111 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 23815ef8124e6f55115a4c3d96238fa9 in 444ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:10,115 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextException/23815ef8124e6f55115a4c3d96238fa9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:10,116 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:42:10,116 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23815ef8124e6f55115a4c3d96238fa9: Waiting for close lock at 1732790529667Running coprocessor pre-close hooks at 1732790529667Disabling compacts and flushes for region at 1732790529667Disabling writes for close at 1732790529667Obtaining lock to block concurrent updates at 1732790529667Preparing flush snapshotting stores in 23815ef8124e6f55115a4c3d96238fa9 at 1732790529667Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790529668 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. at 1732790529668Flushing 23815ef8124e6f55115a4c3d96238fa9/testFamily: creating writer at 1732790529668Flushing 23815ef8124e6f55115a4c3d96238fa9/testFamily: appending metadata at 1732790529684 (+16 ms)Flushing 23815ef8124e6f55115a4c3d96238fa9/testFamily: closing flushed file at 1732790529685 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d39112a: reopening flushed file at 1732790530102 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 23815ef8124e6f55115a4c3d96238fa9 in 444ms, sequenceid=8, compaction requested=false at 1732790530111 (+9 ms)Writing region close event to WAL at 1732790530111Running coprocessor post-close hooks at 1732790530116 (+5 ms)Closed at 1732790530116 2024-11-28T10:42:10,116 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732790484849.23815ef8124e6f55115a4c3d96238fa9. 2024-11-28T10:42:10,116 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 94a73c4fa948020f43325f8b1c60ff43, disabling compactions & flushes 2024-11-28T10:42:10,117 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:42:10,117 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:42:10,117 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. after waiting 0 ms 2024-11-28T10:42:10,117 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:42:10,117 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 94a73c4fa948020f43325f8b1c60ff43 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:10,134 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43/.tmp/testFamily/60646da10fec46129015c23519f02666 is 61, key is row-0/testFamily:testQualifier/1732790516346/Put/seqid=0 2024-11-28T10:42:10,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741862_1038 (size=5049) 2024-11-28T10:42:10,237 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:10,437 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, 94a73c4fa948020f43325f8b1c60ff43, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:10,540 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43/.tmp/testFamily/60646da10fec46129015c23519f02666 2024-11-28T10:42:10,548 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43/.tmp/testFamily/60646da10fec46129015c23519f02666 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43/testFamily/60646da10fec46129015c23519f02666 2024-11-28T10:42:10,558 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43/testFamily/60646da10fec46129015c23519f02666, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:10,559 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 94a73c4fa948020f43325f8b1c60ff43 in 442ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:10,564 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNext/94a73c4fa948020f43325f8b1c60ff43/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:10,565 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:42:10,565 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 94a73c4fa948020f43325f8b1c60ff43: Waiting for close lock at 1732790530116Running coprocessor pre-close hooks at 1732790530116Disabling compacts and flushes for region at 1732790530116Disabling writes for close at 1732790530117 (+1 ms)Obtaining lock to block concurrent updates at 1732790530117Preparing flush snapshotting stores in 94a73c4fa948020f43325f8b1c60ff43 at 1732790530117Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790530117Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. at 1732790530118 (+1 ms)Flushing 94a73c4fa948020f43325f8b1c60ff43/testFamily: creating writer at 1732790530118Flushing 94a73c4fa948020f43325f8b1c60ff43/testFamily: appending metadata at 1732790530134 (+16 ms)Flushing 94a73c4fa948020f43325f8b1c60ff43/testFamily: closing flushed file at 1732790530134Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22993cf: reopening flushed file at 1732790530547 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 94a73c4fa948020f43325f8b1c60ff43 in 442ms, sequenceid=8, compaction requested=false at 1732790530559 (+12 ms)Writing region close event to WAL at 1732790530560 (+1 ms)Running coprocessor post-close hooks at 1732790530565 (+5 ms)Closed at 1732790530565 2024-11-28T10:42:10,566 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732790515321.94a73c4fa948020f43325f8b1c60ff43. 2024-11-28T10:42:10,566 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2ec26e43dfdf13098af7fe3044ccf73c, disabling compactions & flushes 2024-11-28T10:42:10,566 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:10,566 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:10,566 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. after waiting 0 ms 2024-11-28T10:42:10,566 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:10,566 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2ec26e43dfdf13098af7fe3044ccf73c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:10,586 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c/.tmp/testFamily/43f15bdfcd824b4db9306c986680113d is 61, key is row-0/testFamily:testQualifier/1732790525453/Put/seqid=0 2024-11-28T10:42:10,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741863_1039 (size=5049) 2024-11-28T10:42:10,638 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-28T10:42:10,638 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1325): Online Regions={2ec26e43dfdf13098af7fe3044ccf73c=testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., e99096e2f5934a127dfc49eb28ac61c6=testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6., e7e15bd2cf2816daa3a7922c2d73578c=hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c., bc5ffa67250885deceb033d7389a32ed=testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed., 61054584bf641bf46d8f877b534d70c1=testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.} 2024-11-28T10:42:10,638 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:10,712 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-28T10:42:10,712 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T10:42:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T10:42:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-28T10:42:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-28T10:42:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-28T10:42:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T10:42:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-28T10:42:10,714 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-28T10:42:10,838 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 2ec26e43dfdf13098af7fe3044ccf73c, 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:10,994 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c/.tmp/testFamily/43f15bdfcd824b4db9306c986680113d 2024-11-28T10:42:11,003 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c/.tmp/testFamily/43f15bdfcd824b4db9306c986680113d as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c/testFamily/43f15bdfcd824b4db9306c986680113d 2024-11-28T10:42:11,011 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c/testFamily/43f15bdfcd824b4db9306c986680113d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:11,012 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ec26e43dfdf13098af7fe3044ccf73c in 446ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:11,018 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnOpenScannerAsync/2ec26e43dfdf13098af7fe3044ccf73c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:11,019 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:11,019 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2ec26e43dfdf13098af7fe3044ccf73c: Waiting for close lock at 1732790530566Running coprocessor pre-close hooks at 1732790530566Disabling compacts and flushes for region at 1732790530566Disabling writes for close at 1732790530566Obtaining lock to block concurrent updates at 1732790530566Preparing flush snapshotting stores in 2ec26e43dfdf13098af7fe3044ccf73c at 1732790530566Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790530566Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. at 1732790530567 (+1 ms)Flushing 2ec26e43dfdf13098af7fe3044ccf73c/testFamily: creating writer at 1732790530567Flushing 2ec26e43dfdf13098af7fe3044ccf73c/testFamily: appending metadata at 1732790530586 (+19 ms)Flushing 2ec26e43dfdf13098af7fe3044ccf73c/testFamily: closing flushed file at 1732790530586Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@406d2e48: reopening flushed file at 1732790531002 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2ec26e43dfdf13098af7fe3044ccf73c in 446ms, sequenceid=8, compaction requested=false at 1732790531012 (+10 ms)Writing region close event to WAL at 1732790531013 (+1 ms)Running coprocessor post-close hooks at 1732790531018 (+5 ms)Closed at 1732790531018 2024-11-28T10:42:11,019 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732790522394.2ec26e43dfdf13098af7fe3044ccf73c. 2024-11-28T10:42:11,019 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e99096e2f5934a127dfc49eb28ac61c6, disabling compactions & flushes 2024-11-28T10:42:11,019 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:42:11,019 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:42:11,019 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. after waiting 0 ms 2024-11-28T10:42:11,019 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:42:11,019 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e99096e2f5934a127dfc49eb28ac61c6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:11,035 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6/.tmp/testFamily/d9e75dbd609049bbbc72ff75050bb188 is 61, key is row-0/testFamily:testQualifier/1732790466559/Put/seqid=0 2024-11-28T10:42:11,038 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:11,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741864_1040 (size=5049) 2024-11-28T10:42:11,239 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:11,439 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c, e99096e2f5934a127dfc49eb28ac61c6 2024-11-28T10:42:11,443 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6/.tmp/testFamily/d9e75dbd609049bbbc72ff75050bb188 2024-11-28T10:42:11,453 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6/.tmp/testFamily/d9e75dbd609049bbbc72ff75050bb188 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6/testFamily/d9e75dbd609049bbbc72ff75050bb188 2024-11-28T10:42:11,460 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6/testFamily/d9e75dbd609049bbbc72ff75050bb188, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:11,461 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e99096e2f5934a127dfc49eb28ac61c6 in 442ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:11,466 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e99096e2f5934a127dfc49eb28ac61c6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:11,467 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:42:11,467 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e99096e2f5934a127dfc49eb28ac61c6: Waiting for close lock at 1732790531019Running coprocessor pre-close hooks at 1732790531019Disabling compacts and flushes for region at 1732790531019Disabling writes for close at 1732790531019Obtaining lock to block concurrent updates at 1732790531019Preparing flush snapshotting stores in e99096e2f5934a127dfc49eb28ac61c6 at 1732790531019Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790531020 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. at 1732790531021 (+1 ms)Flushing e99096e2f5934a127dfc49eb28ac61c6/testFamily: creating writer at 1732790531021Flushing e99096e2f5934a127dfc49eb28ac61c6/testFamily: appending metadata at 1732790531035 (+14 ms)Flushing e99096e2f5934a127dfc49eb28ac61c6/testFamily: closing flushed file at 1732790531035Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75b8fb4d: reopening flushed file at 1732790531452 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e99096e2f5934a127dfc49eb28ac61c6 in 442ms, sequenceid=8, compaction requested=false at 1732790531461 (+9 ms)Writing region close event to WAL at 1732790531462 (+1 ms)Running coprocessor post-close hooks at 1732790531467 (+5 ms)Closed at 1732790531467 2024-11-28T10:42:11,467 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732790465478.e99096e2f5934a127dfc49eb28ac61c6. 2024-11-28T10:42:11,467 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e7e15bd2cf2816daa3a7922c2d73578c, disabling compactions & flushes 2024-11-28T10:42:11,467 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:42:11,467 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:42:11,467 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. after waiting 0 ms 2024-11-28T10:42:11,467 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:42:11,467 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e7e15bd2cf2816daa3a7922c2d73578c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:11,484 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c/.tmp/testFamily/351f556af51a482f8a96de2d0860d0ab is 61, key is row-0/testFamily:testQualifier/1732790478805/Put/seqid=0 2024-11-28T10:42:11,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741865_1041 (size=5049) 2024-11-28T10:42:11,640 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-28T10:42:11,640 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1325): Online Regions={e7e15bd2cf2816daa3a7922c2d73578c=hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c., bc5ffa67250885deceb033d7389a32ed=testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed., 61054584bf641bf46d8f877b534d70c1=testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.} 2024-11-28T10:42:11,640 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:42:11,840 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed, e7e15bd2cf2816daa3a7922c2d73578c 2024-11-28T10:42:11,891 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c/.tmp/testFamily/351f556af51a482f8a96de2d0860d0ab 2024-11-28T10:42:11,904 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c/.tmp/testFamily/351f556af51a482f8a96de2d0860d0ab as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c/testFamily/351f556af51a482f8a96de2d0860d0ab 2024-11-28T10:42:11,911 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c/testFamily/351f556af51a482f8a96de2d0860d0ab, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:11,912 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e7e15bd2cf2816daa3a7922c2d73578c in 445ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:11,917 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/hbase/testMetaScanTimeoutOnNextAsync/e7e15bd2cf2816daa3a7922c2d73578c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:11,918 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:42:11,918 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e7e15bd2cf2816daa3a7922c2d73578c: Waiting for close lock at 1732790531467Running coprocessor pre-close hooks at 1732790531467Disabling compacts and flushes for region at 1732790531467Disabling writes for close at 1732790531467Obtaining lock to block concurrent updates at 1732790531467Preparing flush snapshotting stores in e7e15bd2cf2816daa3a7922c2d73578c at 1732790531467Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790531468 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. at 1732790531468Flushing e7e15bd2cf2816daa3a7922c2d73578c/testFamily: creating writer at 1732790531468Flushing e7e15bd2cf2816daa3a7922c2d73578c/testFamily: appending metadata at 1732790531484 (+16 ms)Flushing e7e15bd2cf2816daa3a7922c2d73578c/testFamily: closing flushed file at 1732790531484Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a491dce: reopening flushed file at 1732790531903 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e7e15bd2cf2816daa3a7922c2d73578c in 445ms, sequenceid=8, compaction requested=false at 1732790531912 (+9 ms)Writing region close event to WAL at 1732790531913 (+1 ms)Running coprocessor post-close hooks at 1732790531917 (+4 ms)Closed at 1732790531917 2024-11-28T10:42:11,918 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732790475759.e7e15bd2cf2816daa3a7922c2d73578c. 2024-11-28T10:42:11,918 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bc5ffa67250885deceb033d7389a32ed, disabling compactions & flushes 2024-11-28T10:42:11,918 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:42:11,918 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:42:11,918 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. after waiting 0 ms 2024-11-28T10:42:11,918 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:42:11,918 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bc5ffa67250885deceb033d7389a32ed 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:11,933 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed/.tmp/testFamily/5b70983027674279a4f413e89fe3c130 is 61, key is row-0/testFamily:testQualifier/1732790497080/Put/seqid=0 2024-11-28T10:42:11,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741866_1042 (size=5049) 2024-11-28T10:42:12,041 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:42:12,241 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1, bc5ffa67250885deceb033d7389a32ed 2024-11-28T10:42:12,341 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed/.tmp/testFamily/5b70983027674279a4f413e89fe3c130 2024-11-28T10:42:12,356 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed/.tmp/testFamily/5b70983027674279a4f413e89fe3c130 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed/testFamily/5b70983027674279a4f413e89fe3c130 2024-11-28T10:42:12,363 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed/testFamily/5b70983027674279a4f413e89fe3c130, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:12,364 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bc5ffa67250885deceb033d7389a32ed in 446ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:12,368 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNextAsync/bc5ffa67250885deceb033d7389a32ed/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:12,369 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:42:12,369 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bc5ffa67250885deceb033d7389a32ed: Waiting for close lock at 1732790531918Running coprocessor pre-close hooks at 1732790531918Disabling compacts and flushes for region at 1732790531918Disabling writes for close at 1732790531918Obtaining lock to block concurrent updates at 1732790531918Preparing flush snapshotting stores in bc5ffa67250885deceb033d7389a32ed at 1732790531918Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790531918Flushing stores of testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. at 1732790531919 (+1 ms)Flushing bc5ffa67250885deceb033d7389a32ed/testFamily: creating writer at 1732790531919Flushing bc5ffa67250885deceb033d7389a32ed/testFamily: appending metadata at 1732790531933 (+14 ms)Flushing bc5ffa67250885deceb033d7389a32ed/testFamily: closing flushed file at 1732790531933Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be4541c: reopening flushed file at 1732790532355 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bc5ffa67250885deceb033d7389a32ed in 446ms, sequenceid=8, compaction requested=false at 1732790532364 (+9 ms)Writing region close event to WAL at 1732790532365 (+1 ms)Running coprocessor post-close hooks at 1732790532369 (+4 ms)Closed at 1732790532369 2024-11-28T10:42:12,369 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732790494033.bc5ffa67250885deceb033d7389a32ed. 2024-11-28T10:42:12,369 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 61054584bf641bf46d8f877b534d70c1, disabling compactions & flushes 2024-11-28T10:42:12,369 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:42:12,369 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:42:12,369 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. after waiting 0 ms 2024-11-28T10:42:12,369 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:42:12,369 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 61054584bf641bf46d8f877b534d70c1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T10:42:12,386 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1/.tmp/testFamily/323b3a08ae7e41aaa59e70ae2cf5da1c is 61, key is row-0/testFamily:testQualifier/1732790503178/Put/seqid=0 2024-11-28T10:42:12,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741867_1043 (size=5049) 2024-11-28T10:42:12,441 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:42:12,642 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-28T10:42:12,642 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1325): Online Regions={61054584bf641bf46d8f877b534d70c1=testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1.} 2024-11-28T10:42:12,642 DEBUG [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1351): Waiting on 61054584bf641bf46d8f877b534d70c1 2024-11-28T10:42:12,793 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1/.tmp/testFamily/323b3a08ae7e41aaa59e70ae2cf5da1c 2024-11-28T10:42:12,804 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1/.tmp/testFamily/323b3a08ae7e41aaa59e70ae2cf5da1c as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1/testFamily/323b3a08ae7e41aaa59e70ae2cf5da1c 2024-11-28T10:42:12,812 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1/testFamily/323b3a08ae7e41aaa59e70ae2cf5da1c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T10:42:12,813 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 61054584bf641bf46d8f877b534d70c1 in 444ms, sequenceid=8, compaction requested=false 2024-11-28T10:42:12,818 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/data/default/testNormalScanTimeoutOnNext/61054584bf641bf46d8f877b534d70c1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T10:42:12,819 INFO [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:42:12,819 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 61054584bf641bf46d8f877b534d70c1: Waiting for close lock at 1732790532369Running coprocessor pre-close hooks at 1732790532369Disabling compacts and flushes for region at 1732790532369Disabling writes for close at 1732790532369Obtaining lock to block concurrent updates at 1732790532369Preparing flush snapshotting stores in 61054584bf641bf46d8f877b534d70c1 at 1732790532369Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732790532370 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. at 1732790532370Flushing 61054584bf641bf46d8f877b534d70c1/testFamily: creating writer at 1732790532370Flushing 61054584bf641bf46d8f877b534d70c1/testFamily: appending metadata at 1732790532385 (+15 ms)Flushing 61054584bf641bf46d8f877b534d70c1/testFamily: closing flushed file at 1732790532386 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77860762: reopening flushed file at 1732790532803 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 61054584bf641bf46d8f877b534d70c1 in 444ms, sequenceid=8, compaction requested=false at 1732790532813 (+10 ms)Writing region close event to WAL at 1732790532814 (+1 ms)Running coprocessor post-close hooks at 1732790532819 (+5 ms)Closed at 1732790532819 2024-11-28T10:42:12,819 DEBUG [RS_CLOSE_REGION-regionserver/e792e7e95410:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732790500119.61054584bf641bf46d8f877b534d70c1. 2024-11-28T10:42:12,843 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(976): stopping server e792e7e95410,34367,1732790431492; all regions closed. 2024-11-28T10:42:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741834_1010 (size=19479) 2024-11-28T10:42:12,858 DEBUG [RS:0;e792e7e95410:34367 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/oldWALs 2024-11-28T10:42:12,858 INFO [RS:0;e792e7e95410:34367 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e792e7e95410%2C34367%2C1732790431492.meta:.meta(num 1732790465071) 2024-11-28T10:42:12,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741833_1009 (size=20031) 2024-11-28T10:42:12,862 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/WALs/e792e7e95410,34367,1732790431492/e792e7e95410%2C34367%2C1732790431492.1732790464662 not finished, retry = 0 2024-11-28T10:42:12,967 DEBUG [RS:0;e792e7e95410:34367 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/oldWALs 2024-11-28T10:42:12,967 INFO [RS:0;e792e7e95410:34367 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e792e7e95410%2C34367%2C1732790431492:(num 1732790464662) 2024-11-28T10:42:12,967 DEBUG [RS:0;e792e7e95410:34367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T10:42:12,967 INFO [RS:0;e792e7e95410:34367 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T10:42:12,968 INFO [RS:0;e792e7e95410:34367 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T10:42:12,968 INFO [RS:0;e792e7e95410:34367 {}] hbase.ChoreService(370): Chore service for: regionserver/e792e7e95410:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T10:42:12,969 INFO [RS:0;e792e7e95410:34367 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T10:42:12,969 INFO [regionserver/e792e7e95410:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T10:42:12,971 INFO [RS:0;e792e7e95410:34367 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34367 2024-11-28T10:42:13,019 INFO [regionserver/e792e7e95410:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T10:42:13,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e792e7e95410,34367,1732790431492 2024-11-28T10:42:13,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T10:42:13,041 INFO [RS:0;e792e7e95410:34367 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T10:42:13,044 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e792e7e95410,34367,1732790431492] 2024-11-28T10:42:13,062 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e792e7e95410,34367,1732790431492 already deleted, retry=false 2024-11-28T10:42:13,063 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e792e7e95410,34367,1732790431492 expired; onlineServers=0 2024-11-28T10:42:13,063 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e792e7e95410,42151,1732790430369' ***** 2024-11-28T10:42:13,063 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T10:42:13,064 INFO [M:0;e792e7e95410:42151 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T10:42:13,064 INFO [M:0;e792e7e95410:42151 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T10:42:13,064 DEBUG [M:0;e792e7e95410:42151 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T10:42:13,065 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T10:42:13,065 DEBUG [M:0;e792e7e95410:42151 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T10:42:13,065 DEBUG [master/e792e7e95410:0:becomeActiveMaster-HFileCleaner.small.0-1732790434488 {}] cleaner.HFileCleaner(306): Exit Thread[master/e792e7e95410:0:becomeActiveMaster-HFileCleaner.small.0-1732790434488,5,FailOnTimeoutGroup] 2024-11-28T10:42:13,065 DEBUG [master/e792e7e95410:0:becomeActiveMaster-HFileCleaner.large.0-1732790434488 {}] cleaner.HFileCleaner(306): Exit Thread[master/e792e7e95410:0:becomeActiveMaster-HFileCleaner.large.0-1732790434488,5,FailOnTimeoutGroup] 2024-11-28T10:42:13,066 INFO [M:0;e792e7e95410:42151 {}] hbase.ChoreService(370): Chore service for: master/e792e7e95410:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T10:42:13,066 INFO [M:0;e792e7e95410:42151 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T10:42:13,066 DEBUG [M:0;e792e7e95410:42151 {}] master.HMaster(1795): Stopping service threads 2024-11-28T10:42:13,066 INFO [M:0;e792e7e95410:42151 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T10:42:13,067 INFO [M:0;e792e7e95410:42151 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T10:42:13,068 INFO [M:0;e792e7e95410:42151 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T10:42:13,068 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T10:42:13,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T10:42:13,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T10:42:13,073 DEBUG [M:0;e792e7e95410:42151 {}] zookeeper.ZKUtil(347): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T10:42:13,073 WARN [M:0;e792e7e95410:42151 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T10:42:13,074 INFO [M:0;e792e7e95410:42151 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/.lastflushedseqids 2024-11-28T10:42:13,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741868_1044 (size=721) 2024-11-28T10:42:13,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T10:42:13,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34367-0x1018107a3380001, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T10:42:13,153 INFO [RS:0;e792e7e95410:34367 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T10:42:13,154 INFO [RS:0;e792e7e95410:34367 {}] regionserver.HRegionServer(1031): Exiting; stopping=e792e7e95410,34367,1732790431492; zookeeper connection closed. 2024-11-28T10:42:13,155 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1c990491 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1c990491 2024-11-28T10:42:13,155 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T10:42:13,491 INFO [M:0;e792e7e95410:42151 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T10:42:13,492 INFO [M:0;e792e7e95410:42151 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T10:42:13,493 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T10:42:13,493 INFO [M:0;e792e7e95410:42151 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T10:42:13,493 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T10:42:13,493 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T10:42:13,494 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T10:42:13,495 INFO [M:0;e792e7e95410:42151 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.19 KB heapSize=191.21 KB 2024-11-28T10:42:13,520 DEBUG [M:0;e792e7e95410:42151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/73517b46c9734daba20ba6cb7e581d2e is 82, key is hbase:meta,,1/info:regioninfo/1732790465155/Put/seqid=0 2024-11-28T10:42:13,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741869_1045 (size=5672) 2024-11-28T10:42:13,927 INFO [M:0;e792e7e95410:42151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/73517b46c9734daba20ba6cb7e581d2e 2024-11-28T10:42:13,962 DEBUG [M:0;e792e7e95410:42151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/951f725b2b21484f984b8fb44ace41d9 is 813, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732790466345/Put/seqid=0 2024-11-28T10:42:13,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741870_1046 (size=14079) 2024-11-28T10:42:14,270 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T10:42:14,368 INFO [M:0;e792e7e95410:42151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.64 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/951f725b2b21484f984b8fb44ace41d9 2024-11-28T10:42:14,379 INFO [M:0;e792e7e95410:42151 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 951f725b2b21484f984b8fb44ace41d9 2024-11-28T10:42:14,398 DEBUG [M:0;e792e7e95410:42151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/faa1baed8ebf4d55b9f5471665085122 is 69, key is e792e7e95410,34367,1732790431492/rs:state/1732790434525/Put/seqid=0 2024-11-28T10:42:14,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741871_1047 (size=5156) 2024-11-28T10:42:14,805 INFO [M:0;e792e7e95410:42151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/faa1baed8ebf4d55b9f5471665085122 2024-11-28T10:42:14,820 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/73517b46c9734daba20ba6cb7e581d2e as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/73517b46c9734daba20ba6cb7e581d2e 2024-11-28T10:42:14,827 INFO [M:0;e792e7e95410:42151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/73517b46c9734daba20ba6cb7e581d2e, entries=8, sequenceid=319, filesize=5.5 K 2024-11-28T10:42:14,828 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/951f725b2b21484f984b8fb44ace41d9 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/951f725b2b21484f984b8fb44ace41d9 2024-11-28T10:42:14,835 INFO [M:0;e792e7e95410:42151 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 951f725b2b21484f984b8fb44ace41d9 2024-11-28T10:42:14,835 INFO [M:0;e792e7e95410:42151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/951f725b2b21484f984b8fb44ace41d9, entries=33, sequenceid=319, filesize=13.7 K 2024-11-28T10:42:14,836 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/faa1baed8ebf4d55b9f5471665085122 as hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/faa1baed8ebf4d55b9f5471665085122 2024-11-28T10:42:14,843 INFO [M:0;e792e7e95410:42151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33827/user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/faa1baed8ebf4d55b9f5471665085122, entries=1, sequenceid=319, filesize=5.0 K 2024-11-28T10:42:14,844 INFO [M:0;e792e7e95410:42151 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.19 KB/167111, heapSize ~190.91 KB/195496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1350ms, sequenceid=319, compaction requested=false 2024-11-28T10:42:14,846 INFO [M:0;e792e7e95410:42151 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T10:42:14,846 DEBUG [M:0;e792e7e95410:42151 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732790533493Disabling compacts and flushes for region at 1732790533493Disabling writes for close at 1732790533493Obtaining lock to block concurrent updates at 1732790533495 (+2 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732790533495Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167111, getHeapSize=195736, getOffHeapSize=0, getCellsCount=366 at 1732790533496 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732790533498 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732790533498Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732790533520 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732790533520Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732790533939 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732790533961 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732790533961Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732790534379 (+418 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732790534397 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732790534397Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8548e99: reopening flushed file at 1732790534818 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@298fb7cb: reopening flushed file at 1732790534827 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fcc8f0a: reopening flushed file at 1732790534835 (+8 ms)Finished flush of dataSize ~163.19 KB/167111, heapSize ~190.91 KB/195496, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1350ms, sequenceid=319, compaction requested=false at 1732790534844 (+9 ms)Writing region close event to WAL at 1732790534846 (+2 ms)Closed at 1732790534846 2024-11-28T10:42:14,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38247 is added to blk_1073741830_1006 (size=188453) 2024-11-28T10:42:14,849 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/5b426e2d-e2eb-ba29-634f-3069e6f44226/MasterData/WALs/e792e7e95410,42151,1732790430369/e792e7e95410%2C42151%2C1732790430369.1732790433787 not finished, retry = 0 2024-11-28T10:42:14,950 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T10:42:14,950 INFO [M:0;e792e7e95410:42151 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T10:42:14,950 INFO [M:0;e792e7e95410:42151 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42151 2024-11-28T10:42:14,950 INFO [M:0;e792e7e95410:42151 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T10:42:15,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T10:42:15,109 INFO [M:0;e792e7e95410:42151 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T10:42:15,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42151-0x1018107a3380000, quorum=127.0.0.1:62053, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T10:42:15,114 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@258822a0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T10:42:15,116 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@514e86a2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T10:42:15,116 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T10:42:15,116 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c1ee62c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T10:42:15,117 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d304465{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/hadoop.log.dir/,STOPPED} 2024-11-28T10:42:15,119 WARN [BP-1945411675-172.17.0.2-1732790426683 heartbeating to localhost/127.0.0.1:33827 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T10:42:15,119 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T10:42:15,119 WARN [BP-1945411675-172.17.0.2-1732790426683 heartbeating to localhost/127.0.0.1:33827 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1945411675-172.17.0.2-1732790426683 (Datanode Uuid 11f46c8d-8f38-42be-87d8-261561bf6700) service to localhost/127.0.0.1:33827 2024-11-28T10:42:15,119 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T10:42:15,120 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/cluster_148ccf20-f6ae-26ef-ff5e-8273dc2c0562/data/data1/current/BP-1945411675-172.17.0.2-1732790426683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T10:42:15,120 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/cluster_148ccf20-f6ae-26ef-ff5e-8273dc2c0562/data/data2/current/BP-1945411675-172.17.0.2-1732790426683 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T10:42:15,121 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T10:42:15,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@59bae7b3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T10:42:15,132 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f237650{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T10:42:15,132 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T10:42:15,132 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@45f63f41{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T10:42:15,132 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@660c56f7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/ffb4fe8f-7074-cdf3-4f87-d373e7bafb99/hadoop.log.dir/,STOPPED} 2024-11-28T10:42:15,140 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T10:42:15,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down