2024-11-28 03:57:05,303 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-28 03:57:05,317 main DEBUG Took 0.011927 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-28 03:57:05,317 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-28 03:57:05,318 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-28 03:57:05,319 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-28 03:57:05,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,347 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-28 03:57:05,362 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,364 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,366 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,366 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,367 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,368 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,369 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,369 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,370 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,370 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,371 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,372 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,372 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,373 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,374 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,374 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,376 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,377 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-28 03:57:05,378 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,379 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-28 03:57:05,381 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-28 03:57:05,383 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-28 03:57:05,385 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-28 03:57:05,386 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-28 03:57:05,388 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-28 03:57:05,388 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-28 03:57:05,397 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-28 03:57:05,400 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-28 03:57:05,402 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-28 03:57:05,402 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-28 03:57:05,403 main DEBUG createAppenders(={Console}) 2024-11-28 03:57:05,404 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-28 03:57:05,404 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-28 03:57:05,404 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-28 03:57:05,405 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-28 03:57:05,406 main DEBUG OutputStream closed 2024-11-28 03:57:05,406 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-28 03:57:05,406 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-28 03:57:05,407 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-28 03:57:05,483 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-28 03:57:05,485 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-28 03:57:05,486 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-28 03:57:05,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-28 03:57:05,487 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-28 03:57:05,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-28 03:57:05,488 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-28 03:57:05,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-28 03:57:05,489 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-28 03:57:05,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-28 03:57:05,490 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-28 03:57:05,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-28 03:57:05,491 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-28 03:57:05,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-28 03:57:05,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-28 03:57:05,492 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-28 03:57:05,493 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-28 03:57:05,493 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-28 03:57:05,496 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28 03:57:05,496 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-28 03:57:05,496 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-28 03:57:05,497 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-28T03:57:05,514 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-28 03:57:05,517 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-28 03:57:05,517 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-28T03:57:05,859 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64 2024-11-28T03:57:05,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-28T03:57:05,899 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/cluster_7e20e934-fbf9-def3-20eb-b2cc66e44915, deleteOnExit=true 2024-11-28T03:57:05,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-28T03:57:05,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/test.cache.data in system properties and HBase conf 2024-11-28T03:57:05,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/hadoop.tmp.dir in system properties and HBase conf 2024-11-28T03:57:05,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/hadoop.log.dir in system properties and HBase conf 2024-11-28T03:57:05,902 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-28T03:57:05,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-28T03:57:05,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-28T03:57:06,014 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-28T03:57:06,115 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-28T03:57:06,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-28T03:57:06,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-28T03:57:06,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-28T03:57:06,122 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T03:57:06,123 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-28T03:57:06,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-28T03:57:06,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-28T03:57:06,124 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T03:57:06,125 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-28T03:57:06,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/nfs.dump.dir in system properties and HBase conf 2024-11-28T03:57:06,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/java.io.tmpdir in system properties and HBase conf 2024-11-28T03:57:06,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-28T03:57:06,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-28T03:57:06,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-28T03:57:07,042 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-28T03:57:07,125 INFO [Time-limited test {}] log.Log(170): Logging initialized @2661ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-28T03:57:07,209 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T03:57:07,288 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T03:57:07,311 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T03:57:07,312 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T03:57:07,313 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-28T03:57:07,326 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T03:57:07,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/hadoop.log.dir/,AVAILABLE} 2024-11-28T03:57:07,330 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T03:57:07,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/java.io.tmpdir/jetty-localhost-45609-hadoop-hdfs-3_4_1-tests_jar-_-any-2669245697668215344/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T03:57:07,602 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:45609} 2024-11-28T03:57:07,602 INFO [Time-limited test {}] server.Server(415): Started @3138ms 2024-11-28T03:57:08,009 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-28T03:57:08,016 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-28T03:57:08,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-28T03:57:08,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-28T03:57:08,017 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-28T03:57:08,018 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@662cf64e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/hadoop.log.dir/,AVAILABLE} 2024-11-28T03:57:08,019 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@23178064{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-28T03:57:08,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7722c06d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/java.io.tmpdir/jetty-localhost-44765-hadoop-hdfs-3_4_1-tests_jar-_-any-9714003233973277276/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T03:57:08,170 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@445877d1{HTTP/1.1, (http/1.1)}{localhost:44765} 2024-11-28T03:57:08,170 INFO [Time-limited test {}] server.Server(415): Started @3707ms 2024-11-28T03:57:08,251 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-28T03:57:08,766 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/cluster_7e20e934-fbf9-def3-20eb-b2cc66e44915/data/data2/current/BP-1691337829-172.17.0.2-1732766226774/current, will proceed with Du for space computation calculation, 2024-11-28T03:57:08,766 WARN [Thread-70 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/cluster_7e20e934-fbf9-def3-20eb-b2cc66e44915/data/data1/current/BP-1691337829-172.17.0.2-1732766226774/current, will proceed with Du for space computation calculation, 2024-11-28T03:57:08,842 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-28T03:57:08,901 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a3fa37b00e8d0c7 with lease ID 0xf4be30f7406a6606: Processing first storage report for DS-b094e8ed-25ae-4fff-9aa6-9e855f7cb18e from datanode DatanodeRegistration(127.0.0.1:39619, datanodeUuid=da0b1283-7ca7-4081-83e0-b9a6b833dbdb, infoPort=43565, infoSecurePort=0, ipcPort=43973, storageInfo=lv=-57;cid=testClusterID;nsid=883509850;c=1732766226774) 2024-11-28T03:57:08,902 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a3fa37b00e8d0c7 with lease ID 0xf4be30f7406a6606: from storage DS-b094e8ed-25ae-4fff-9aa6-9e855f7cb18e node DatanodeRegistration(127.0.0.1:39619, datanodeUuid=da0b1283-7ca7-4081-83e0-b9a6b833dbdb, infoPort=43565, infoSecurePort=0, ipcPort=43973, storageInfo=lv=-57;cid=testClusterID;nsid=883509850;c=1732766226774), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-28T03:57:08,903 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a3fa37b00e8d0c7 with lease ID 0xf4be30f7406a6606: Processing first storage report for DS-c3047055-3c9c-4929-9830-57bcecad4ffd from datanode DatanodeRegistration(127.0.0.1:39619, datanodeUuid=da0b1283-7ca7-4081-83e0-b9a6b833dbdb, infoPort=43565, infoSecurePort=0, ipcPort=43973, storageInfo=lv=-57;cid=testClusterID;nsid=883509850;c=1732766226774) 2024-11-28T03:57:08,903 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a3fa37b00e8d0c7 with lease ID 0xf4be30f7406a6606: from storage DS-c3047055-3c9c-4929-9830-57bcecad4ffd node DatanodeRegistration(127.0.0.1:39619, datanodeUuid=da0b1283-7ca7-4081-83e0-b9a6b833dbdb, infoPort=43565, infoSecurePort=0, ipcPort=43973, storageInfo=lv=-57;cid=testClusterID;nsid=883509850;c=1732766226774), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-28T03:57:08,953 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64 2024-11-28T03:57:09,051 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/cluster_7e20e934-fbf9-def3-20eb-b2cc66e44915/zookeeper_0, clientPort=64005, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/cluster_7e20e934-fbf9-def3-20eb-b2cc66e44915/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/cluster_7e20e934-fbf9-def3-20eb-b2cc66e44915/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-28T03:57:09,063 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64005 2024-11-28T03:57:09,075 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:57:09,079 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:57:09,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741825_1001 (size=7) 2024-11-28T03:57:09,780 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd with version=8 2024-11-28T03:57:09,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/hbase-staging 2024-11-28T03:57:09,889 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-28T03:57:10,179 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3a546af21488:0 server-side Connection retries=9 2024-11-28T03:57:10,192 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T03:57:10,192 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T03:57:10,197 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T03:57:10,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T03:57:10,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T03:57:10,365 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-28T03:57:10,442 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-28T03:57:10,451 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-28T03:57:10,456 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T03:57:10,494 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1919 (auto-detected) 2024-11-28T03:57:10,495 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-28T03:57:10,521 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45807 2024-11-28T03:57:10,546 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45807 connecting to ZooKeeper ensemble=127.0.0.1:64005 2024-11-28T03:57:10,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:458070x0, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T03:57:10,585 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45807-0x1003c461aff0000 connected 2024-11-28T03:57:10,619 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:57:10,622 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:57:10,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T03:57:10,636 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd, hbase.cluster.distributed=false 2024-11-28T03:57:10,668 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T03:57:10,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45807 2024-11-28T03:57:10,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45807 2024-11-28T03:57:10,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45807 2024-11-28T03:57:10,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45807 2024-11-28T03:57:10,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45807 2024-11-28T03:57:10,812 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3a546af21488:0 server-side Connection retries=9 2024-11-28T03:57:10,814 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T03:57:10,814 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-28T03:57:10,814 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-28T03:57:10,814 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-28T03:57:10,814 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-28T03:57:10,817 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-28T03:57:10,820 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-28T03:57:10,821 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36023 2024-11-28T03:57:10,823 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36023 connecting to ZooKeeper ensemble=127.0.0.1:64005 2024-11-28T03:57:10,825 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:57:10,829 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:57:10,839 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360230x0, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-28T03:57:10,840 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36023-0x1003c461aff0001 connected 2024-11-28T03:57:10,840 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T03:57:10,846 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-28T03:57:10,858 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-28T03:57:10,861 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-28T03:57:10,866 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-28T03:57:10,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36023 2024-11-28T03:57:10,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36023 2024-11-28T03:57:10,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36023 2024-11-28T03:57:10,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36023 2024-11-28T03:57:10,872 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36023 2024-11-28T03:57:10,888 DEBUG [M:0;3a546af21488:45807 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3a546af21488:45807 2024-11-28T03:57:10,889 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3a546af21488,45807,1732766229951 2024-11-28T03:57:10,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T03:57:10,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T03:57:10,900 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3a546af21488,45807,1732766229951 2024-11-28T03:57:10,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-28T03:57:10,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:10,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:10,922 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-28T03:57:10,923 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3a546af21488,45807,1732766229951 from backup master directory 2024-11-28T03:57:10,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T03:57:10,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3a546af21488,45807,1732766229951 2024-11-28T03:57:10,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-28T03:57:10,928 WARN [master/3a546af21488:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T03:57:10,928 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3a546af21488,45807,1732766229951 2024-11-28T03:57:10,931 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-28T03:57:10,932 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-28T03:57:10,996 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/hbase.id] with ID: 19b5f47f-0c4a-41e6-9466-1e14e7afeb01 2024-11-28T03:57:10,996 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/.tmp/hbase.id 2024-11-28T03:57:11,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741826_1002 (size=42) 2024-11-28T03:57:11,410 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/.tmp/hbase.id]:[hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/hbase.id] 2024-11-28T03:57:11,460 INFO [master/3a546af21488:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-28T03:57:11,465 INFO [master/3a546af21488:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-28T03:57:11,485 INFO [master/3a546af21488:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-28T03:57:11,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:11,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:11,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741827_1003 (size=196) 2024-11-28T03:57:11,524 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:57:11,526 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-28T03:57:11,545 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:57:11,550 INFO [master/3a546af21488:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T03:57:11,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741828_1004 (size=1189) 2024-11-28T03:57:11,603 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store 2024-11-28T03:57:11,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741829_1005 (size=34) 2024-11-28T03:57:12,029 INFO [master/3a546af21488:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-28T03:57:12,034 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:57:12,036 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T03:57:12,037 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:57:12,037 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:57:12,040 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T03:57:12,040 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:57:12,040 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:57:12,042 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732766232036Disabling compacts and flushes for region at 1732766232036Disabling writes for close at 1732766232040 (+4 ms)Writing region close event to WAL at 1732766232040Closed at 1732766232040 2024-11-28T03:57:12,045 WARN [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/.initializing 2024-11-28T03:57:12,046 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/WALs/3a546af21488,45807,1732766229951 2024-11-28T03:57:12,058 INFO [master/3a546af21488:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T03:57:12,076 INFO [master/3a546af21488:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a546af21488%2C45807%2C1732766229951, suffix=, logDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/WALs/3a546af21488,45807,1732766229951, archiveDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/oldWALs, maxLogs=10 2024-11-28T03:57:12,114 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/WALs/3a546af21488,45807,1732766229951/3a546af21488%2C45807%2C1732766229951.1732766232083, exclude list is [], retry=0 2024-11-28T03:57:12,134 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39619,DS-b094e8ed-25ae-4fff-9aa6-9e855f7cb18e,DISK] 2024-11-28T03:57:12,137 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-28T03:57:12,180 INFO [master/3a546af21488:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/WALs/3a546af21488,45807,1732766229951/3a546af21488%2C45807%2C1732766229951.1732766232083 2024-11-28T03:57:12,181 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43565:43565)] 2024-11-28T03:57:12,182 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:57:12,183 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:57:12,187 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,188 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,264 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-28T03:57:12,268 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:12,270 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:57:12,271 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,274 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-28T03:57:12,275 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:12,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:57:12,276 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,279 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-28T03:57:12,280 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:12,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:57:12,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,284 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-28T03:57:12,285 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:12,286 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:57:12,286 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,291 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,292 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,299 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,300 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,305 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T03:57:12,309 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-28T03:57:12,314 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:57:12,316 INFO [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70386258, jitterRate=0.048836976289749146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T03:57:12,323 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732766232205Initializing all the Stores at 1732766232207 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732766232208 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766232209 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766232209Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766232209Cleaning up temporary data from old regions at 1732766232300 (+91 ms)Region opened successfully at 1732766232322 (+22 ms) 2024-11-28T03:57:12,324 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-28T03:57:12,361 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@268a5ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3a546af21488/172.17.0.2:0 2024-11-28T03:57:12,396 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-28T03:57:12,408 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-28T03:57:12,408 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-28T03:57:12,412 INFO [master/3a546af21488:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-28T03:57:12,413 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-28T03:57:12,418 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-28T03:57:12,418 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-28T03:57:12,445 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-28T03:57:12,453 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-28T03:57:12,455 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-28T03:57:12,458 INFO [master/3a546af21488:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-28T03:57:12,459 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-28T03:57:12,461 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-28T03:57:12,464 INFO [master/3a546af21488:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-28T03:57:12,468 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-28T03:57:12,470 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-28T03:57:12,471 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-28T03:57:12,473 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-28T03:57:12,493 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-28T03:57:12,496 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-28T03:57:12,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T03:57:12,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-28T03:57:12,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:12,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:12,503 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3a546af21488,45807,1732766229951, sessionid=0x1003c461aff0000, setting cluster-up flag (Was=false) 2024-11-28T03:57:12,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:12,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:12,524 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-28T03:57:12,526 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3a546af21488,45807,1732766229951 2024-11-28T03:57:12,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:12,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:57:12,546 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-28T03:57:12,548 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3a546af21488,45807,1732766229951 2024-11-28T03:57:12,555 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-28T03:57:12,578 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(746): ClusterId : 19b5f47f-0c4a-41e6-9466-1e14e7afeb01 2024-11-28T03:57:12,582 DEBUG [RS:0;3a546af21488:36023 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-28T03:57:12,588 DEBUG [RS:0;3a546af21488:36023 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-28T03:57:12,588 DEBUG [RS:0;3a546af21488:36023 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-28T03:57:12,591 DEBUG [RS:0;3a546af21488:36023 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-28T03:57:12,592 DEBUG [RS:0;3a546af21488:36023 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38642c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3a546af21488/172.17.0.2:0 2024-11-28T03:57:12,610 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3a546af21488:36023 2024-11-28T03:57:12,614 INFO [RS:0;3a546af21488:36023 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-28T03:57:12,615 INFO [RS:0;3a546af21488:36023 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-28T03:57:12,615 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-28T03:57:12,618 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(2659): reportForDuty to master=3a546af21488,45807,1732766229951 with port=36023, startcode=1732766230768 2024-11-28T03:57:12,630 DEBUG [RS:0;3a546af21488:36023 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-28T03:57:12,648 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-28T03:57:12,664 INFO [master/3a546af21488:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-28T03:57:12,673 INFO [master/3a546af21488:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-28T03:57:12,674 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39425, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-28T03:57:12,682 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45807 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T03:57:12,683 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3a546af21488,45807,1732766229951 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-28T03:57:12,692 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3a546af21488:0, corePoolSize=5, maxPoolSize=5 2024-11-28T03:57:12,692 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3a546af21488:0, corePoolSize=5, maxPoolSize=5 2024-11-28T03:57:12,692 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3a546af21488:0, corePoolSize=5, maxPoolSize=5 2024-11-28T03:57:12,692 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3a546af21488:0, corePoolSize=5, maxPoolSize=5 2024-11-28T03:57:12,693 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3a546af21488:0, corePoolSize=10, maxPoolSize=10 2024-11-28T03:57:12,693 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:12,693 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3a546af21488:0, corePoolSize=2, maxPoolSize=2 2024-11-28T03:57:12,693 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:12,701 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T03:57:12,701 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-28T03:57:12,709 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732766262709 2024-11-28T03:57:12,710 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:12,710 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-28T03:57:12,711 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-28T03:57:12,713 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-28T03:57:12,714 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-28T03:57:12,715 WARN [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-28T03:57:12,717 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-28T03:57:12,718 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-28T03:57:12,718 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-28T03:57:12,718 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-28T03:57:12,719 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:12,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741831_1007 (size=1321) 2024-11-28T03:57:12,724 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-28T03:57:12,724 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-28T03:57:12,725 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:57:12,726 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-28T03:57:12,726 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-28T03:57:12,730 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-28T03:57:12,731 INFO [master/3a546af21488:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-28T03:57:12,733 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3a546af21488:0:becomeActiveMaster-HFileCleaner.large.0-1732766232732,5,FailOnTimeoutGroup] 2024-11-28T03:57:12,734 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3a546af21488:0:becomeActiveMaster-HFileCleaner.small.0-1732766232733,5,FailOnTimeoutGroup] 2024-11-28T03:57:12,734 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:12,734 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-28T03:57:12,736 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:12,736 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:12,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741832_1008 (size=32) 2024-11-28T03:57:13,139 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:57:13,142 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T03:57:13,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T03:57:13,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:13,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:57:13,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T03:57:13,151 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T03:57:13,151 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:13,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:57:13,152 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T03:57:13,154 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T03:57:13,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:13,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:57:13,156 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T03:57:13,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T03:57:13,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:57:13,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:57:13,159 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T03:57:13,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740 2024-11-28T03:57:13,161 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740 2024-11-28T03:57:13,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T03:57:13,165 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T03:57:13,166 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T03:57:13,168 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T03:57:13,173 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:57:13,174 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65175284, jitterRate=-0.02881258726119995}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T03:57:13,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732766233139Initializing all the Stores at 1732766233142 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732766233142Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732766233142Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766233142Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732766233142Cleaning up temporary data from old regions at 1732766233165 (+23 ms)Region opened successfully at 1732766233177 (+12 ms) 2024-11-28T03:57:13,177 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T03:57:13,178 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T03:57:13,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T03:57:13,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T03:57:13,178 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T03:57:13,180 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T03:57:13,180 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732766233177Disabling compacts and flushes for region at 1732766233177Disabling writes for close at 1732766233178 (+1 ms)Writing region close event to WAL at 1732766233179 (+1 ms)Closed at 1732766233180 (+1 ms) 2024-11-28T03:57:13,184 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T03:57:13,184 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-28T03:57:13,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-28T03:57:13,200 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T03:57:13,204 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-28T03:57:13,355 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:14,355 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:15,356 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:16,357 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:17,357 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:18,358 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:19,359 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:20,359 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:20,439 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-28T03:57:20,439 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-28T03:57:21,360 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:22,361 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:23,362 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:24,362 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:25,363 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:26,364 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:27,364 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:28,365 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:29,366 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:30,366 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:31,367 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:32,368 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:33,368 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:34,369 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:35,370 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:36,371 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:37,371 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:38,372 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:38,948 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:57:39,373 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:40,373 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:41,374 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:42,374 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:42,716 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(2659): reportForDuty to master=3a546af21488,45807,1732766229951 with port=36023, startcode=1732766230768 2024-11-28T03:57:42,718 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45807 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3a546af21488,36023,1732766230768 2024-11-28T03:57:42,721 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45807 {}] master.ServerManager(517): Registering regionserver=3a546af21488,36023,1732766230768 2024-11-28T03:57:42,731 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:57:42,731 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46373 2024-11-28T03:57:42,731 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-28T03:57:42,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T03:57:42,736 DEBUG [RS:0;3a546af21488:36023 {}] zookeeper.ZKUtil(111): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3a546af21488,36023,1732766230768 2024-11-28T03:57:42,736 WARN [RS:0;3a546af21488:36023 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-28T03:57:42,737 INFO [RS:0;3a546af21488:36023 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T03:57:42,737 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/WALs/3a546af21488,36023,1732766230768 2024-11-28T03:57:42,739 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3a546af21488,36023,1732766230768] 2024-11-28T03:57:42,763 INFO [RS:0;3a546af21488:36023 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-28T03:57:42,774 INFO [RS:0;3a546af21488:36023 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-28T03:57:42,779 INFO [RS:0;3a546af21488:36023 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-28T03:57:42,779 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,780 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-28T03:57:42,786 INFO [RS:0;3a546af21488:36023 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-28T03:57:42,787 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,787 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,788 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,788 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,788 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,788 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,788 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3a546af21488:0, corePoolSize=2, maxPoolSize=2 2024-11-28T03:57:42,788 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,788 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,789 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,789 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,789 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,789 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3a546af21488:0, corePoolSize=1, maxPoolSize=1 2024-11-28T03:57:42,789 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3a546af21488:0, corePoolSize=3, maxPoolSize=3 2024-11-28T03:57:42,789 DEBUG [RS:0;3a546af21488:36023 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3a546af21488:0, corePoolSize=3, maxPoolSize=3 2024-11-28T03:57:42,790 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,790 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,790 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,791 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,791 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,791 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,36023,1732766230768-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T03:57:42,816 INFO [RS:0;3a546af21488:36023 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-28T03:57:42,818 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,36023,1732766230768-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,819 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,819 INFO [RS:0;3a546af21488:36023 {}] regionserver.Replication(171): 3a546af21488,36023,1732766230768 started 2024-11-28T03:57:42,839 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:57:42,839 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1482): Serving as 3a546af21488,36023,1732766230768, RpcServer on 3a546af21488/172.17.0.2:36023, sessionid=0x1003c461aff0001 2024-11-28T03:57:42,841 DEBUG [RS:0;3a546af21488:36023 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-28T03:57:42,841 DEBUG [RS:0;3a546af21488:36023 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3a546af21488,36023,1732766230768 2024-11-28T03:57:42,841 DEBUG [RS:0;3a546af21488:36023 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3a546af21488,36023,1732766230768' 2024-11-28T03:57:42,841 DEBUG [RS:0;3a546af21488:36023 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-28T03:57:42,843 DEBUG [RS:0;3a546af21488:36023 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-28T03:57:42,843 DEBUG [RS:0;3a546af21488:36023 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-28T03:57:42,844 DEBUG [RS:0;3a546af21488:36023 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-28T03:57:42,844 DEBUG [RS:0;3a546af21488:36023 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3a546af21488,36023,1732766230768 2024-11-28T03:57:42,844 DEBUG [RS:0;3a546af21488:36023 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3a546af21488,36023,1732766230768' 2024-11-28T03:57:42,844 DEBUG [RS:0;3a546af21488:36023 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-28T03:57:42,845 DEBUG [RS:0;3a546af21488:36023 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-28T03:57:42,845 DEBUG [RS:0;3a546af21488:36023 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-28T03:57:42,846 INFO [RS:0;3a546af21488:36023 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-28T03:57:42,846 INFO [RS:0;3a546af21488:36023 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-28T03:57:43,375 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:44,376 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:45,376 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:46,377 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:47,378 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:48,378 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:49,379 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:50,379 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:51,380 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:52,381 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:53,381 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:54,382 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:55,383 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:56,383 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:57,384 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:58,385 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:57:59,385 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:00,386 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:01,387 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:02,388 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:03,388 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:04,389 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:05,390 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:06,390 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:07,391 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:08,392 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:08,949 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:58:09,392 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:10,393 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:11,394 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:12,394 WARN [3a546af21488:45807 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-28T03:58:12,851 INFO [RS:0;3a546af21488:36023 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-28T03:58:12,856 INFO [RS:0;3a546af21488:36023 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a546af21488%2C36023%2C1732766230768, suffix=, logDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/WALs/3a546af21488,36023,1732766230768, archiveDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/oldWALs, maxLogs=32 2024-11-28T03:58:12,874 DEBUG [RS:0;3a546af21488:36023 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/WALs/3a546af21488,36023,1732766230768/3a546af21488%2C36023%2C1732766230768.1732766292858, exclude list is [], retry=0 2024-11-28T03:58:12,880 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39619,DS-b094e8ed-25ae-4fff-9aa6-9e855f7cb18e,DISK] 2024-11-28T03:58:12,883 INFO [RS:0;3a546af21488:36023 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/WALs/3a546af21488,36023,1732766230768/3a546af21488%2C36023%2C1732766230768.1732766292858 2024-11-28T03:58:12,884 DEBUG [RS:0;3a546af21488:36023 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43565:43565)] 2024-11-28T03:58:13,147 DEBUG [3a546af21488:45807 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-28T03:58:13,162 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:13,171 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3a546af21488,36023,1732766230768, state=OPENING 2024-11-28T03:58:13,176 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-28T03:58:13,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:58:13,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:58:13,179 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T03:58:13,179 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T03:58:13,181 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-28T03:58:13,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:13,364 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-28T03:58:13,367 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55785, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-28T03:58:13,381 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-28T03:58:13,381 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-28T03:58:13,382 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-28T03:58:13,386 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3a546af21488%2C36023%2C1732766230768.meta, suffix=.meta, logDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/WALs/3a546af21488,36023,1732766230768, archiveDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/oldWALs, maxLogs=32 2024-11-28T03:58:13,405 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/WALs/3a546af21488,36023,1732766230768/3a546af21488%2C36023%2C1732766230768.meta.1732766293388.meta, exclude list is [], retry=0 2024-11-28T03:58:13,410 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39619,DS-b094e8ed-25ae-4fff-9aa6-9e855f7cb18e,DISK] 2024-11-28T03:58:13,414 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/WALs/3a546af21488,36023,1732766230768/3a546af21488%2C36023%2C1732766230768.meta.1732766293388.meta 2024-11-28T03:58:13,415 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43565:43565)] 2024-11-28T03:58:13,415 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:13,417 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-28T03:58:13,421 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-28T03:58:13,428 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-28T03:58:13,434 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-28T03:58:13,435 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:13,435 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-28T03:58:13,435 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-28T03:58:13,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-28T03:58:13,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-28T03:58:13,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:13,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:58:13,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-28T03:58:13,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-28T03:58:13,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:13,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:58:13,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-28T03:58:13,447 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-28T03:58:13,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:13,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:58:13,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-28T03:58:13,450 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-28T03:58:13,450 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:13,451 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-28T03:58:13,451 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-28T03:58:13,453 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740 2024-11-28T03:58:13,455 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740 2024-11-28T03:58:13,458 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-28T03:58:13,458 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-28T03:58:13,459 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-28T03:58:13,463 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-28T03:58:13,465 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66333896, jitterRate=-0.01154792308807373}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-28T03:58:13,465 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-28T03:58:13,467 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732766293436Writing region info on filesystem at 1732766293436Initializing all the Stores at 1732766293439 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732766293439Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732766293439Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766293439Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732766293439Cleaning up temporary data from old regions at 1732766293458 (+19 ms)Running coprocessor post-open hooks at 1732766293465 (+7 ms)Region opened successfully at 1732766293467 (+2 ms) 2024-11-28T03:58:13,478 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732766293355 2024-11-28T03:58:13,493 DEBUG [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-28T03:58:13,493 INFO [RS_OPEN_META-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-28T03:58:13,496 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:13,498 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3a546af21488,36023,1732766230768, state=OPEN 2024-11-28T03:58:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T03:58:13,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-28T03:58:13,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T03:58:13,504 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3a546af21488,36023,1732766230768 2024-11-28T03:58:13,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-28T03:58:13,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-28T03:58:13,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3a546af21488,36023,1732766230768 in 321 msec 2024-11-28T03:58:13,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-28T03:58:13,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.322 sec 2024-11-28T03:58:13,520 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-28T03:58:13,520 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-28T03:58:13,543 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:58:13,545 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,36023,1732766230768, seqNum=-1] 2024-11-28T03:58:13,569 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:58:13,573 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49761, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:58:13,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 1.009 sec 2024-11-28T03:58:13,600 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732766293599, completionTime=-1 2024-11-28T03:58:13,604 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-28T03:58:13,604 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-28T03:58:13,640 INFO [master/3a546af21488:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-28T03:58:13,640 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732766353640 2024-11-28T03:58:13,641 INFO [master/3a546af21488:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732766413640 2024-11-28T03:58:13,641 INFO [master/3a546af21488:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-11-28T03:58:13,643 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,45807,1732766229951-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-28T03:58:13,644 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,45807,1732766229951-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:58:13,644 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,45807,1732766229951-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:58:13,646 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3a546af21488:45807, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:58:13,646 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-28T03:58:13,647 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-28T03:58:13,655 DEBUG [master/3a546af21488:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-28T03:58:13,680 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 62.751sec 2024-11-28T03:58:13,681 INFO [master/3a546af21488:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-28T03:58:13,683 INFO [master/3a546af21488:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-28T03:58:13,684 INFO [master/3a546af21488:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-28T03:58:13,684 INFO [master/3a546af21488:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-28T03:58:13,684 INFO [master/3a546af21488:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-28T03:58:13,685 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,45807,1732766229951-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-28T03:58:13,686 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,45807,1732766229951-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-28T03:58:13,694 DEBUG [master/3a546af21488:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-28T03:58:13,694 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-28T03:58:13,695 INFO [master/3a546af21488:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3a546af21488,45807,1732766229951-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-28T03:58:13,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49293b90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:58:13,743 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-28T03:58:13,743 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-28T03:58:13,747 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,45807,-1 for getting cluster id 2024-11-28T03:58:13,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:58:13,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b5f47f-0c4a-41e6-9466-1e14e7afeb01' 2024-11-28T03:58:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:58:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b5f47f-0c4a-41e6-9466-1e14e7afeb01" 2024-11-28T03:58:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287d24fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:58:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,45807,-1] 2024-11-28T03:58:13,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:58:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:58:13,771 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:58:13,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36b1a32d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:58:13,774 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:58:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,36023,1732766230768, seqNum=-1] 2024-11-28T03:58:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:58:13,784 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:58:13,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3a546af21488,45807,1732766229951 2024-11-28T03:58:13,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@589b6ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:58:13,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3a546af21488,45807,-1 for getting cluster id 2024-11-28T03:58:13,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-28T03:58:13,854 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '19b5f47f-0c4a-41e6-9466-1e14e7afeb01' 2024-11-28T03:58:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-28T03:58:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "19b5f47f-0c4a-41e6-9466-1e14e7afeb01" 2024-11-28T03:58:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51eae32c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:58:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3a546af21488,45807,-1] 2024-11-28T03:58:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-28T03:58:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:58:13,865 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-28T03:58:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30c842d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-28T03:58:13,890 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=6759 2024-11-28T03:58:13,897 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-28T03:58:13,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3a546af21488,45807,1732766229951 2024-11-28T03:58:13,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18689943 2024-11-28T03:58:13,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-28T03:58:13,910 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51730, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-28T03:58:13,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-28T03:58:13,931 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:13,935 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:13,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-28T03:58:13,938 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:13,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T03:58:13,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741835_1011 (size=428) 2024-11-28T03:58:14,357 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e7155b0e7763de9cbc461b8ae4b4836, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:14,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741836_1012 (size=79) 2024-11-28T03:58:14,370 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:14,370 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 4e7155b0e7763de9cbc461b8ae4b4836, disabling compactions & flushes 2024-11-28T03:58:14,370 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:58:14,370 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:58:14,370 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. after waiting 0 ms 2024-11-28T03:58:14,371 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:58:14,371 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:58:14,371 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e7155b0e7763de9cbc461b8ae4b4836: Waiting for close lock at 1732766294370Disabling compacts and flushes for region at 1732766294370Disabling writes for close at 1732766294371 (+1 ms)Writing region close event to WAL at 1732766294371Closed at 1732766294371 2024-11-28T03:58:14,373 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:14,378 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732766294373"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766294373"}]},"ts":"1732766294373"} 2024-11-28T03:58:14,385 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:14,387 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:14,390 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766294387"}]},"ts":"1732766294387"} 2024-11-28T03:58:14,395 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-28T03:58:14,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e7155b0e7763de9cbc461b8ae4b4836, ASSIGN}] 2024-11-28T03:58:14,400 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e7155b0e7763de9cbc461b8ae4b4836, ASSIGN 2024-11-28T03:58:14,402 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e7155b0e7763de9cbc461b8ae4b4836, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:14,553 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e7155b0e7763de9cbc461b8ae4b4836, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:14,559 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e7155b0e7763de9cbc461b8ae4b4836, ASSIGN because future has completed 2024-11-28T03:58:14,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e7155b0e7763de9cbc461b8ae4b4836, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:14,722 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:58:14,722 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e7155b0e7763de9cbc461b8ae4b4836, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:14,723 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,723 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:14,723 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,724 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,726 INFO [StoreOpener-4e7155b0e7763de9cbc461b8ae4b4836-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,729 INFO [StoreOpener-4e7155b0e7763de9cbc461b8ae4b4836-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e7155b0e7763de9cbc461b8ae4b4836 columnFamilyName testFamily 2024-11-28T03:58:14,729 DEBUG [StoreOpener-4e7155b0e7763de9cbc461b8ae4b4836-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:14,730 INFO [StoreOpener-4e7155b0e7763de9cbc461b8ae4b4836-1 {}] regionserver.HStore(327): Store=4e7155b0e7763de9cbc461b8ae4b4836/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:58:14,730 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,732 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,733 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,733 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,734 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,737 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,748 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:58:14,750 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4e7155b0e7763de9cbc461b8ae4b4836; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71063721, jitterRate=0.05893196165561676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:58:14,750 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:58:14,751 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4e7155b0e7763de9cbc461b8ae4b4836: Running coprocessor pre-open hook at 1732766294724Writing region info on filesystem at 1732766294724Initializing all the Stores at 1732766294726 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766294726Cleaning up temporary data from old regions at 1732766294734 (+8 ms)Running coprocessor post-open hooks at 1732766294750 (+16 ms)Region opened successfully at 1732766294751 (+1 ms) 2024-11-28T03:58:14,754 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836., pid=6, masterSystemTime=1732766294714 2024-11-28T03:58:14,758 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:58:14,758 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:58:14,760 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4e7155b0e7763de9cbc461b8ae4b4836, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:14,764 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e7155b0e7763de9cbc461b8ae4b4836, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:58:14,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-28T03:58:14,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e7155b0e7763de9cbc461b8ae4b4836, server=3a546af21488,36023,1732766230768 in 206 msec 2024-11-28T03:58:14,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-28T03:58:14,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=4e7155b0e7763de9cbc461b8ae4b4836, ASSIGN in 374 msec 2024-11-28T03:58:14,777 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:58:14,777 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766294777"}]},"ts":"1732766294777"} 2024-11-28T03:58:14,781 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-28T03:58:14,782 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:58:14,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 861 msec 2024-11-28T03:58:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-28T03:58:14,959 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-28T03:58:14,959 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-28T03:58:14,961 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:14,967 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-28T03:58:14,968 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:14,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-28T03:58:14,972 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-28T03:58:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3a546af21488,36023,1732766230768, seqNum=-1] 2024-11-28T03:58:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-28T03:58:14,977 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-28T03:58:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:14,999 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:58:15,003 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-28T03:58:15,008 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-28T03:58:15,013 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-28T03:58:15,015 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-28T03:58:15,016 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6135864783863283720 2024-11-28T03:58:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46454 deadline: 1732766298015, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T03:58:15,025 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-28T03:58:15,027 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-28T03:58:15,029 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:15,030 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:15,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6135864783863283722 2024-11-28T03:58:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46454 deadline: 1732766298030, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T03:58:15,035 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:15,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=6135864783863283723 2024-11-28T03:58:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46454 deadline: 1732766298034, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T03:58:15,039 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:15,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=6135864783863283724 2024-11-28T03:58:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46454 deadline: 1732766298039, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T03:58:15,044 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:15,058 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3a546af21488:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1049951891_22 at /127.0.0.1:57976 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=6741 (was 6759) 2024-11-28T03:58:15,068 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=6741 2024-11-28T03:58:15,072 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:15,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-28T03:58:15,077 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:15,078 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:15,078 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-28T03:58:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:58:15,080 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:15,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741837_1013 (size=414) 2024-11-28T03:58:15,093 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7be12718b584945b0f2a51a2da19802a, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:15,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741838_1014 (size=65) 2024-11-28T03:58:15,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:15,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 7be12718b584945b0f2a51a2da19802a, disabling compactions & flushes 2024-11-28T03:58:15,505 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:58:15,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:58:15,505 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. after waiting 0 ms 2024-11-28T03:58:15,506 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:58:15,506 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:58:15,506 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7be12718b584945b0f2a51a2da19802a: Waiting for close lock at 1732766295505Disabling compacts and flushes for region at 1732766295505Disabling writes for close at 1732766295506 (+1 ms)Writing region close event to WAL at 1732766295506Closed at 1732766295506 2024-11-28T03:58:15,511 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:15,511 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732766295511"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766295511"}]},"ts":"1732766295511"} 2024-11-28T03:58:15,516 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:15,520 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:15,520 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766295520"}]},"ts":"1732766295520"} 2024-11-28T03:58:15,525 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-28T03:58:15,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=7be12718b584945b0f2a51a2da19802a, ASSIGN}] 2024-11-28T03:58:15,527 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=7be12718b584945b0f2a51a2da19802a, ASSIGN 2024-11-28T03:58:15,530 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=7be12718b584945b0f2a51a2da19802a, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:15,681 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7be12718b584945b0f2a51a2da19802a, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:15,687 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=7be12718b584945b0f2a51a2da19802a, ASSIGN because future has completed 2024-11-28T03:58:15,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7be12718b584945b0f2a51a2da19802a, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:15,848 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:58:15,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 7be12718b584945b0f2a51a2da19802a, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:15,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:15,850 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,850 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,852 INFO [StoreOpener-7be12718b584945b0f2a51a2da19802a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,855 INFO [StoreOpener-7be12718b584945b0f2a51a2da19802a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7be12718b584945b0f2a51a2da19802a columnFamilyName testFamily 2024-11-28T03:58:15,855 DEBUG [StoreOpener-7be12718b584945b0f2a51a2da19802a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:15,856 INFO [StoreOpener-7be12718b584945b0f2a51a2da19802a-1 {}] regionserver.HStore(327): Store=7be12718b584945b0f2a51a2da19802a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:58:15,856 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,858 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,858 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,859 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,862 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,867 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:58:15,868 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 7be12718b584945b0f2a51a2da19802a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74638535, jitterRate=0.11220084130764008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:58:15,868 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:58:15,869 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 7be12718b584945b0f2a51a2da19802a: Running coprocessor pre-open hook at 1732766295850Writing region info on filesystem at 1732766295850Initializing all the Stores at 1732766295852 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766295852Cleaning up temporary data from old regions at 1732766295859 (+7 ms)Running coprocessor post-open hooks at 1732766295868 (+9 ms)Region opened successfully at 1732766295869 (+1 ms) 2024-11-28T03:58:15,871 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a., pid=9, masterSystemTime=1732766295842 2024-11-28T03:58:15,876 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:58:15,876 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:58:15,877 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7be12718b584945b0f2a51a2da19802a, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:15,881 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7be12718b584945b0f2a51a2da19802a, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:58:15,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-28T03:58:15,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 7be12718b584945b0f2a51a2da19802a, server=3a546af21488,36023,1732766230768 in 197 msec 2024-11-28T03:58:15,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-28T03:58:15,893 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=7be12718b584945b0f2a51a2da19802a, ASSIGN in 363 msec 2024-11-28T03:58:15,894 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:58:15,895 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766295894"}]},"ts":"1732766295894"} 2024-11-28T03:58:15,898 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-28T03:58:15,899 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:58:15,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 826 msec 2024-11-28T03:58:16,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-28T03:58:16,089 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-28T03:58:16,089 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-28T03:58:16,090 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:16,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-28T03:58:16,095 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:16,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-28T03:58:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:16,111 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:58:16,111 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-28T03:58:16,114 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T03:58:16,116 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-28T03:58:18,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:58:19,000 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-28T03:58:19,002 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-28T03:58:19,004 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-28T03:58:20,438 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-28T03:58:20,438 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-28T03:58:20,440 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T03:58:20,440 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-28T03:58:20,440 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-28T03:58:20,440 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T03:58:20,441 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-28T03:58:20,441 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-28T03:58:22,128 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a., hostname=3a546af21488,36023,1732766230768, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a., hostname=3a546af21488,36023,1732766230768, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a546af21488:36023 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-11-28T03:58:22,129 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a., hostname=3a546af21488,36023,1732766230768, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-11-28T03:58:22,129 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a., hostname=3a546af21488,36023,1732766230768, seqNum=2 from cache 2024-11-28T03:58:22,143 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=299 (was 291) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6718 (was 6741) 2024-11-28T03:58:22,156 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=299, ProcessCount=11, AvailableMemoryMB=6714 2024-11-28T03:58:22,160 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:22,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-28T03:58:22,165 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:22,165 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:22,165 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-28T03:58:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-28T03:58:22,168 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:22,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741839_1015 (size=412) 2024-11-28T03:58:22,183 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd86fb12fcfed3489581169697519e39, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:22,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741840_1016 (size=63) 2024-11-28T03:58:22,195 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:22,196 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing bd86fb12fcfed3489581169697519e39, disabling compactions & flushes 2024-11-28T03:58:22,196 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:58:22,196 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:58:22,196 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. after waiting 0 ms 2024-11-28T03:58:22,196 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:58:22,196 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:58:22,196 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd86fb12fcfed3489581169697519e39: Waiting for close lock at 1732766302196Disabling compacts and flushes for region at 1732766302196Disabling writes for close at 1732766302196Writing region close event to WAL at 1732766302196Closed at 1732766302196 2024-11-28T03:58:22,199 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:22,199 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732766302199"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766302199"}]},"ts":"1732766302199"} 2024-11-28T03:58:22,203 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:22,205 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:22,205 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766302205"}]},"ts":"1732766302205"} 2024-11-28T03:58:22,208 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-28T03:58:22,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=bd86fb12fcfed3489581169697519e39, ASSIGN}] 2024-11-28T03:58:22,210 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=bd86fb12fcfed3489581169697519e39, ASSIGN 2024-11-28T03:58:22,211 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=bd86fb12fcfed3489581169697519e39, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:22,362 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=bd86fb12fcfed3489581169697519e39, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:22,368 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=bd86fb12fcfed3489581169697519e39, ASSIGN because future has completed 2024-11-28T03:58:22,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd86fb12fcfed3489581169697519e39, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:22,530 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:58:22,531 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => bd86fb12fcfed3489581169697519e39, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:22,531 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,531 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:22,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,532 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,534 INFO [StoreOpener-bd86fb12fcfed3489581169697519e39-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,536 INFO [StoreOpener-bd86fb12fcfed3489581169697519e39-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd86fb12fcfed3489581169697519e39 columnFamilyName testFamily 2024-11-28T03:58:22,536 DEBUG [StoreOpener-bd86fb12fcfed3489581169697519e39-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:22,537 INFO [StoreOpener-bd86fb12fcfed3489581169697519e39-1 {}] regionserver.HStore(327): Store=bd86fb12fcfed3489581169697519e39/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:58:22,537 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,539 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,540 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,540 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,540 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,543 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,548 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:58:22,549 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened bd86fb12fcfed3489581169697519e39; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74044256, jitterRate=0.10334539413452148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:58:22,549 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd86fb12fcfed3489581169697519e39 2024-11-28T03:58:22,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for bd86fb12fcfed3489581169697519e39: Running coprocessor pre-open hook at 1732766302532Writing region info on filesystem at 1732766302532Initializing all the Stores at 1732766302534 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766302534Cleaning up temporary data from old regions at 1732766302540 (+6 ms)Running coprocessor post-open hooks at 1732766302549 (+9 ms)Region opened successfully at 1732766302550 (+1 ms) 2024-11-28T03:58:22,552 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39., pid=12, masterSystemTime=1732766302524 2024-11-28T03:58:22,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:58:22,555 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:58:22,557 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=bd86fb12fcfed3489581169697519e39, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:22,560 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd86fb12fcfed3489581169697519e39, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:58:22,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-28T03:58:22,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure bd86fb12fcfed3489581169697519e39, server=3a546af21488,36023,1732766230768 in 193 msec 2024-11-28T03:58:22,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-28T03:58:22,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=bd86fb12fcfed3489581169697519e39, ASSIGN in 357 msec 2024-11-28T03:58:22,570 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:58:22,570 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766302570"}]},"ts":"1732766302570"} 2024-11-28T03:58:22,574 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-28T03:58:22,575 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:58:22,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 415 msec 2024-11-28T03:58:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-28T03:58:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-28T03:58:23,189 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-28T03:58:23,189 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-28T03:58:23,189 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:23,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-28T03:58:23,195 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:23,195 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-28T03:58:23,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:23,214 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:58:23,214 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-28T03:58:23,217 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T03:58:23,221 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T03:58:23,221 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T03:58:23,222 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-28T03:58:25,005 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-28T03:58:28,034 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:58:29,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36023 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283733' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T03:58:29,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36023 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:46454 deadline: 1732766315232, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283733' does not exist 2024-11-28T03:58:29,237 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3a546af21488,36023,1732766230768 for closing scanner id = 6135864783863283733 for bd86fb12fcfed3489581169697519e39 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283733' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:58:29,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 299), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=6537 (was 6714) 2024-11-28T03:58:29,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=12, AvailableMemoryMB=6536 2024-11-28T03:58:29,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-28T03:58:29,258 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:29,258 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:29,258 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-28T03:58:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:58:29,260 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:29,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741841_1017 (size=423) 2024-11-28T03:58:29,678 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5bf8ed37879fb968af0047760657b90a, NAME => 'testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:29,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741842_1018 (size=74) 2024-11-28T03:58:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-28T03:58:30,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:30,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 5bf8ed37879fb968af0047760657b90a, disabling compactions & flushes 2024-11-28T03:58:30,088 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:58:30,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:58:30,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. after waiting 0 ms 2024-11-28T03:58:30,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:58:30,088 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:58:30,088 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5bf8ed37879fb968af0047760657b90a: Waiting for close lock at 1732766310088Disabling compacts and flushes for region at 1732766310088Disabling writes for close at 1732766310088Writing region close event to WAL at 1732766310088Closed at 1732766310088 2024-11-28T03:58:30,090 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:30,090 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732766310090"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766310090"}]},"ts":"1732766310090"} 2024-11-28T03:58:30,093 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:30,095 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:30,095 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766310095"}]},"ts":"1732766310095"} 2024-11-28T03:58:30,097 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-28T03:58:30,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5bf8ed37879fb968af0047760657b90a, ASSIGN}] 2024-11-28T03:58:30,099 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5bf8ed37879fb968af0047760657b90a, ASSIGN 2024-11-28T03:58:30,100 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5bf8ed37879fb968af0047760657b90a, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:30,251 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5bf8ed37879fb968af0047760657b90a, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:30,254 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5bf8ed37879fb968af0047760657b90a, ASSIGN because future has completed 2024-11-28T03:58:30,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bf8ed37879fb968af0047760657b90a, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:58:30,414 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:58:30,415 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 5bf8ed37879fb968af0047760657b90a, NAME => 'testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:30,415 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,415 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:30,415 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,415 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,417 INFO [StoreOpener-5bf8ed37879fb968af0047760657b90a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,421 INFO [StoreOpener-5bf8ed37879fb968af0047760657b90a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5bf8ed37879fb968af0047760657b90a columnFamilyName testFamily 2024-11-28T03:58:30,421 DEBUG [StoreOpener-5bf8ed37879fb968af0047760657b90a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:30,422 INFO [StoreOpener-5bf8ed37879fb968af0047760657b90a-1 {}] regionserver.HStore(327): Store=5bf8ed37879fb968af0047760657b90a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:58:30,422 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,423 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,423 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,424 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,424 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,426 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,429 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:58:30,430 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 5bf8ed37879fb968af0047760657b90a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64768293, jitterRate=-0.0348772257566452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:58:30,430 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:58:30,431 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 5bf8ed37879fb968af0047760657b90a: Running coprocessor pre-open hook at 1732766310415Writing region info on filesystem at 1732766310415Initializing all the Stores at 1732766310417 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766310417Cleaning up temporary data from old regions at 1732766310424 (+7 ms)Running coprocessor post-open hooks at 1732766310430 (+6 ms)Region opened successfully at 1732766310431 (+1 ms) 2024-11-28T03:58:30,432 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a., pid=15, masterSystemTime=1732766310408 2024-11-28T03:58:30,435 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:58:30,435 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:58:30,436 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5bf8ed37879fb968af0047760657b90a, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:30,438 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-28T03:58:30,438 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T03:58:30,439 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5bf8ed37879fb968af0047760657b90a, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:58:30,439 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-28T03:58:30,439 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-28T03:58:30,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-28T03:58:30,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5bf8ed37879fb968af0047760657b90a, server=3a546af21488,36023,1732766230768 in 186 msec 2024-11-28T03:58:30,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-28T03:58:30,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5bf8ed37879fb968af0047760657b90a, ASSIGN in 347 msec 2024-11-28T03:58:30,448 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:58:30,449 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766310449"}]},"ts":"1732766310449"} 2024-11-28T03:58:30,451 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-28T03:58:30,452 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:58:30,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.1980 sec 2024-11-28T03:58:31,006 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-28T03:58:32,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-28T03:58:32,289 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-28T03:58:32,289 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-28T03:58:32,290 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:32,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-28T03:58:32,294 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:32,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-28T03:58:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:32,307 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:58:32,308 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-28T03:58:32,310 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-28T03:58:32,312 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-28T03:58:32,313 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-28T03:58:32,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6135864783863283737 2024-11-28T03:58:32,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46454 deadline: 1732766315312, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T03:58:32,316 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-28T03:58:32,317 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-28T03:58:32,318 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:32,319 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:32,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6135864783863283739 2024-11-28T03:58:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46454 deadline: 1732766315318, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T03:58:32,322 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:32,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=6135864783863283740 2024-11-28T03:58:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46454 deadline: 1732766315321, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T03:58:32,325 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:32,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=6135864783863283741 2024-11-28T03:58:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:46454 deadline: 1732766315325, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-28T03:58:32,328 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-28T03:58:32,337 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1049951891_22 at /127.0.0.1:56218 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 291), ProcessCount=11 (was 12), AvailableMemoryMB=6539 (was 6536) - AvailableMemoryMB LEAK? - 2024-11-28T03:58:32,345 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=284, ProcessCount=11, AvailableMemoryMB=6539 2024-11-28T03:58:32,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-28T03:58:32,351 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:32,351 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:32,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-28T03:58:32,352 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T03:58:32,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741843_1019 (size=418) 2024-11-28T03:58:32,764 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ea5f362764d0f80a2c40406d9057f965, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:32,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741844_1020 (size=69) 2024-11-28T03:58:33,173 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:33,173 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing ea5f362764d0f80a2c40406d9057f965, disabling compactions & flushes 2024-11-28T03:58:33,173 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:58:33,173 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:58:33,173 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. after waiting 0 ms 2024-11-28T03:58:33,173 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:58:33,173 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:58:33,173 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for ea5f362764d0f80a2c40406d9057f965: Waiting for close lock at 1732766313173Disabling compacts and flushes for region at 1732766313173Disabling writes for close at 1732766313173Writing region close event to WAL at 1732766313173Closed at 1732766313173 2024-11-28T03:58:33,176 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:33,176 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732766313176"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766313176"}]},"ts":"1732766313176"} 2024-11-28T03:58:33,179 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:33,181 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:33,181 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766313181"}]},"ts":"1732766313181"} 2024-11-28T03:58:33,184 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-28T03:58:33,185 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ea5f362764d0f80a2c40406d9057f965, ASSIGN}] 2024-11-28T03:58:33,186 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ea5f362764d0f80a2c40406d9057f965, ASSIGN 2024-11-28T03:58:33,188 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ea5f362764d0f80a2c40406d9057f965, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:33,338 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ea5f362764d0f80a2c40406d9057f965, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:33,342 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ea5f362764d0f80a2c40406d9057f965, ASSIGN because future has completed 2024-11-28T03:58:33,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea5f362764d0f80a2c40406d9057f965, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T03:58:33,501 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:58:33,502 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => ea5f362764d0f80a2c40406d9057f965, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:33,502 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,502 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:33,502 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,502 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,504 INFO [StoreOpener-ea5f362764d0f80a2c40406d9057f965-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,506 INFO [StoreOpener-ea5f362764d0f80a2c40406d9057f965-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea5f362764d0f80a2c40406d9057f965 columnFamilyName testFamily 2024-11-28T03:58:33,506 DEBUG [StoreOpener-ea5f362764d0f80a2c40406d9057f965-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:33,507 INFO [StoreOpener-ea5f362764d0f80a2c40406d9057f965-1 {}] regionserver.HStore(327): Store=ea5f362764d0f80a2c40406d9057f965/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:58:33,507 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,508 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,509 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,509 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,509 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,512 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,515 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:58:33,515 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened ea5f362764d0f80a2c40406d9057f965; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71719481, jitterRate=0.0687035471200943}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:58:33,516 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:58:33,516 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for ea5f362764d0f80a2c40406d9057f965: Running coprocessor pre-open hook at 1732766313502Writing region info on filesystem at 1732766313502Initializing all the Stores at 1732766313504 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766313504Cleaning up temporary data from old regions at 1732766313509 (+5 ms)Running coprocessor post-open hooks at 1732766313516 (+7 ms)Region opened successfully at 1732766313516 2024-11-28T03:58:33,518 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., pid=18, masterSystemTime=1732766313495 2024-11-28T03:58:33,521 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:58:33,521 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:58:33,521 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ea5f362764d0f80a2c40406d9057f965, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:33,525 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ea5f362764d0f80a2c40406d9057f965, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:58:33,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-28T03:58:33,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure ea5f362764d0f80a2c40406d9057f965, server=3a546af21488,36023,1732766230768 in 184 msec 2024-11-28T03:58:33,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-28T03:58:33,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ea5f362764d0f80a2c40406d9057f965, ASSIGN in 344 msec 2024-11-28T03:58:33,533 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:58:33,533 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766313533"}]},"ts":"1732766313533"} 2024-11-28T03:58:33,536 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-28T03:58:33,537 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:58:33,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1890 sec 2024-11-28T03:58:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-28T03:58:35,378 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-28T03:58:35,378 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-28T03:58:35,379 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:35,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-28T03:58:35,383 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:35,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-28T03:58:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:35,397 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:58:35,397 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-28T03:58:35,401 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T03:58:35,402 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T03:58:35,917 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:58:36,409 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a546af21488:36023 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-28T03:58:36,409 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-28T03:58:36,409 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2 from cache 2024-11-28T03:58:36,903 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-28T03:58:37,008 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-28T03:58:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:37,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T03:58:38,408 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a546af21488:36023 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=979ms 2024-11-28T03:58:38,408 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=987ms, rpcTimeout=979ms 2024-11-28T03:58:38,408 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., hostname=3a546af21488,36023,1732766230768, seqNum=2 from cache 2024-11-28T03:58:38,418 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=269 (was 284), ProcessCount=11 (was 11), AvailableMemoryMB=6535 (was 6539) 2024-11-28T03:58:38,426 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=269, ProcessCount=11, AvailableMemoryMB=6534 2024-11-28T03:58:38,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-28T03:58:38,432 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:38,432 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:38,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-28T03:58:38,433 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T03:58:38,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741845_1021 (size=416) 2024-11-28T03:58:38,845 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => adba58a68e6b825d2651bbdf5f9778a4, NAME => 'testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:38,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741846_1022 (size=67) 2024-11-28T03:58:38,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-28T03:58:38,949 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:58:39,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:39,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing adba58a68e6b825d2651bbdf5f9778a4, disabling compactions & flushes 2024-11-28T03:58:39,254 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:58:39,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:58:39,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. after waiting 0 ms 2024-11-28T03:58:39,255 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:58:39,255 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:58:39,255 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for adba58a68e6b825d2651bbdf5f9778a4: Waiting for close lock at 1732766319254Disabling compacts and flushes for region at 1732766319254Disabling writes for close at 1732766319255 (+1 ms)Writing region close event to WAL at 1732766319255Closed at 1732766319255 2024-11-28T03:58:39,257 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:39,258 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732766319257"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766319257"}]},"ts":"1732766319257"} 2024-11-28T03:58:39,261 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:39,262 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:39,263 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766319262"}]},"ts":"1732766319262"} 2024-11-28T03:58:39,265 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-28T03:58:39,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=adba58a68e6b825d2651bbdf5f9778a4, ASSIGN}] 2024-11-28T03:58:39,267 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=adba58a68e6b825d2651bbdf5f9778a4, ASSIGN 2024-11-28T03:58:39,268 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=adba58a68e6b825d2651bbdf5f9778a4, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:39,418 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=adba58a68e6b825d2651bbdf5f9778a4, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:39,422 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=adba58a68e6b825d2651bbdf5f9778a4, ASSIGN because future has completed 2024-11-28T03:58:39,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure adba58a68e6b825d2651bbdf5f9778a4, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T03:58:39,581 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:58:39,581 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => adba58a68e6b825d2651bbdf5f9778a4, NAME => 'testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:39,582 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,582 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:39,582 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,582 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,584 INFO [StoreOpener-adba58a68e6b825d2651bbdf5f9778a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,586 INFO [StoreOpener-adba58a68e6b825d2651bbdf5f9778a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region adba58a68e6b825d2651bbdf5f9778a4 columnFamilyName testFamily 2024-11-28T03:58:39,586 DEBUG [StoreOpener-adba58a68e6b825d2651bbdf5f9778a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:39,587 INFO [StoreOpener-adba58a68e6b825d2651bbdf5f9778a4-1 {}] regionserver.HStore(327): Store=adba58a68e6b825d2651bbdf5f9778a4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:58:39,587 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,588 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,589 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,589 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,589 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,592 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,595 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:58:39,596 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened adba58a68e6b825d2651bbdf5f9778a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74382773, jitterRate=0.10838969051837921}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:58:39,596 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:58:39,596 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for adba58a68e6b825d2651bbdf5f9778a4: Running coprocessor pre-open hook at 1732766319582Writing region info on filesystem at 1732766319582Initializing all the Stores at 1732766319584 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766319584Cleaning up temporary data from old regions at 1732766319589 (+5 ms)Running coprocessor post-open hooks at 1732766319596 (+7 ms)Region opened successfully at 1732766319596 2024-11-28T03:58:39,598 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4., pid=21, masterSystemTime=1732766319575 2024-11-28T03:58:39,600 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:58:39,600 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:58:39,601 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=adba58a68e6b825d2651bbdf5f9778a4, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:39,604 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure adba58a68e6b825d2651bbdf5f9778a4, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:58:39,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-28T03:58:39,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure adba58a68e6b825d2651bbdf5f9778a4, server=3a546af21488,36023,1732766230768 in 183 msec 2024-11-28T03:58:39,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-28T03:58:39,611 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=adba58a68e6b825d2651bbdf5f9778a4, ASSIGN in 343 msec 2024-11-28T03:58:39,612 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:58:39,612 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766319612"}]},"ts":"1732766319612"} 2024-11-28T03:58:39,614 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-28T03:58:39,615 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:58:39,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.1870 sec 2024-11-28T03:58:40,438 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-28T03:58:40,438 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-28T03:58:40,439 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-28T03:58:40,439 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T03:58:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-28T03:58:41,459 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-28T03:58:41,459 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-28T03:58:41,460 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:41,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-28T03:58:41,465 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:41,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-28T03:58:41,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:41,480 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:58:41,480 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-28T03:58:41,483 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T03:58:41,485 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T03:58:41,487 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T03:58:41,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-28T03:58:43,009 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-28T03:58:44,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36023 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283752' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T03:58:44,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36023 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:46454 deadline: 1732766325491, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283752' does not exist 2024-11-28T03:58:44,493 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3a546af21488,36023,1732766230768 for closing scanner id = 6135864783863283752 for adba58a68e6b825d2651bbdf5f9778a4 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283752' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:58:44,498 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=255 (was 269), ProcessCount=11 (was 11), AvailableMemoryMB=6530 (was 6534) 2024-11-28T03:58:44,506 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=6530 2024-11-28T03:58:44,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-28T03:58:44,514 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:44,515 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:44,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-28T03:58:44,516 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T03:58:44,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741847_1023 (size=411) 2024-11-28T03:58:44,927 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 646cc9bf476050085baf95e2500a1dfc, NAME => 'testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:44,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741848_1024 (size=62) 2024-11-28T03:58:44,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-28T03:58:45,083 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:58:45,337 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:45,337 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 646cc9bf476050085baf95e2500a1dfc, disabling compactions & flushes 2024-11-28T03:58:45,337 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:58:45,337 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:58:45,337 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. after waiting 0 ms 2024-11-28T03:58:45,337 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:58:45,337 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:58:45,337 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 646cc9bf476050085baf95e2500a1dfc: Waiting for close lock at 1732766325337Disabling compacts and flushes for region at 1732766325337Disabling writes for close at 1732766325337Writing region close event to WAL at 1732766325337Closed at 1732766325337 2024-11-28T03:58:45,339 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:45,339 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732766325339"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766325339"}]},"ts":"1732766325339"} 2024-11-28T03:58:45,342 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:45,344 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:45,344 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766325344"}]},"ts":"1732766325344"} 2024-11-28T03:58:45,346 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-28T03:58:45,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=646cc9bf476050085baf95e2500a1dfc, ASSIGN}] 2024-11-28T03:58:45,348 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=646cc9bf476050085baf95e2500a1dfc, ASSIGN 2024-11-28T03:58:45,349 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=646cc9bf476050085baf95e2500a1dfc, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:45,500 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=646cc9bf476050085baf95e2500a1dfc, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:45,504 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=646cc9bf476050085baf95e2500a1dfc, ASSIGN because future has completed 2024-11-28T03:58:45,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 646cc9bf476050085baf95e2500a1dfc, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T03:58:45,663 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:58:45,663 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 646cc9bf476050085baf95e2500a1dfc, NAME => 'testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:45,663 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,664 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:45,664 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,664 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,666 INFO [StoreOpener-646cc9bf476050085baf95e2500a1dfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,668 INFO [StoreOpener-646cc9bf476050085baf95e2500a1dfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 646cc9bf476050085baf95e2500a1dfc columnFamilyName testFamily 2024-11-28T03:58:45,668 DEBUG [StoreOpener-646cc9bf476050085baf95e2500a1dfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:45,668 INFO [StoreOpener-646cc9bf476050085baf95e2500a1dfc-1 {}] regionserver.HStore(327): Store=646cc9bf476050085baf95e2500a1dfc/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:58:45,669 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,670 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,670 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,671 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,671 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,673 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,675 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:58:45,676 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 646cc9bf476050085baf95e2500a1dfc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68944933, jitterRate=0.027359560132026672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:58:45,676 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:58:45,677 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 646cc9bf476050085baf95e2500a1dfc: Running coprocessor pre-open hook at 1732766325664Writing region info on filesystem at 1732766325664Initializing all the Stores at 1732766325665 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766325665Cleaning up temporary data from old regions at 1732766325671 (+6 ms)Running coprocessor post-open hooks at 1732766325676 (+5 ms)Region opened successfully at 1732766325677 (+1 ms) 2024-11-28T03:58:45,678 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc., pid=24, masterSystemTime=1732766325657 2024-11-28T03:58:45,681 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:58:45,681 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:58:45,682 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=646cc9bf476050085baf95e2500a1dfc, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:45,685 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 646cc9bf476050085baf95e2500a1dfc, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:58:45,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-28T03:58:45,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 646cc9bf476050085baf95e2500a1dfc, server=3a546af21488,36023,1732766230768 in 182 msec 2024-11-28T03:58:45,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-28T03:58:45,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=646cc9bf476050085baf95e2500a1dfc, ASSIGN in 343 msec 2024-11-28T03:58:45,693 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:58:45,694 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766325693"}]},"ts":"1732766325693"} 2024-11-28T03:58:45,697 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-28T03:58:45,698 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:58:45,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1870 sec 2024-11-28T03:58:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-28T03:58:47,548 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-28T03:58:47,548 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-28T03:58:47,549 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:47,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-28T03:58:47,553 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:47,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-28T03:58:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:47,567 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:58:47,567 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-28T03:58:47,570 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T03:58:47,572 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T03:58:47,573 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T03:58:47,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-28T03:58:49,010 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-28T03:58:50,438 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-28T03:58:50,438 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-28T03:58:50,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36023 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283756' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T03:58:50,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36023 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:46454 deadline: 1732766331582, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283756' does not exist 2024-11-28T03:58:50,583 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3a546af21488,36023,1732766230768 for closing scanner id = 6135864783863283756 for 646cc9bf476050085baf95e2500a1dfc of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283756' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:58:50,588 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 255), ProcessCount=11 (was 11), AvailableMemoryMB=6527 (was 6530) 2024-11-28T03:58:50,596 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=6527 2024-11-28T03:58:50,599 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:50,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-28T03:58:50,602 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:50,602 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:50,602 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-28T03:58:50,603 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:58:50,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741849_1025 (size=419) 2024-11-28T03:58:51,013 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0dd90cc2721494b37accfc7c4d1bcf35, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:51,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741850_1026 (size=70) 2024-11-28T03:58:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-28T03:58:51,165 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:58:51,422 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:51,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0dd90cc2721494b37accfc7c4d1bcf35, disabling compactions & flushes 2024-11-28T03:58:51,423 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:58:51,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:58:51,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. after waiting 0 ms 2024-11-28T03:58:51,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:58:51,423 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:58:51,423 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0dd90cc2721494b37accfc7c4d1bcf35: Waiting for close lock at 1732766331423Disabling compacts and flushes for region at 1732766331423Disabling writes for close at 1732766331423Writing region close event to WAL at 1732766331423Closed at 1732766331423 2024-11-28T03:58:51,425 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:51,425 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732766331425"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766331425"}]},"ts":"1732766331425"} 2024-11-28T03:58:51,428 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:51,429 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:51,429 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766331429"}]},"ts":"1732766331429"} 2024-11-28T03:58:51,431 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-28T03:58:51,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0dd90cc2721494b37accfc7c4d1bcf35, ASSIGN}] 2024-11-28T03:58:51,433 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0dd90cc2721494b37accfc7c4d1bcf35, ASSIGN 2024-11-28T03:58:51,434 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0dd90cc2721494b37accfc7c4d1bcf35, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:51,585 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=0dd90cc2721494b37accfc7c4d1bcf35, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:51,588 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0dd90cc2721494b37accfc7c4d1bcf35, ASSIGN because future has completed 2024-11-28T03:58:51,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dd90cc2721494b37accfc7c4d1bcf35, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:58:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:58:51,747 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:58:51,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 0dd90cc2721494b37accfc7c4d1bcf35, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:58:51,747 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,748 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:51,748 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,748 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,750 INFO [StoreOpener-0dd90cc2721494b37accfc7c4d1bcf35-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,752 INFO [StoreOpener-0dd90cc2721494b37accfc7c4d1bcf35-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0dd90cc2721494b37accfc7c4d1bcf35 columnFamilyName testFamily 2024-11-28T03:58:51,752 DEBUG [StoreOpener-0dd90cc2721494b37accfc7c4d1bcf35-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:51,753 INFO [StoreOpener-0dd90cc2721494b37accfc7c4d1bcf35-1 {}] regionserver.HStore(327): Store=0dd90cc2721494b37accfc7c4d1bcf35/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:58:51,753 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,754 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,754 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,761 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:58:51,761 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 0dd90cc2721494b37accfc7c4d1bcf35; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71601770, jitterRate=0.0669495165348053}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:58:51,762 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:58:51,763 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 0dd90cc2721494b37accfc7c4d1bcf35: Running coprocessor pre-open hook at 1732766331748Writing region info on filesystem at 1732766331748Initializing all the Stores at 1732766331749 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766331749Cleaning up temporary data from old regions at 1732766331756 (+7 ms)Running coprocessor post-open hooks at 1732766331762 (+6 ms)Region opened successfully at 1732766331763 (+1 ms) 2024-11-28T03:58:51,764 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35., pid=27, masterSystemTime=1732766331741 2024-11-28T03:58:51,767 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:58:51,767 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:58:51,767 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=0dd90cc2721494b37accfc7c4d1bcf35, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:51,770 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0dd90cc2721494b37accfc7c4d1bcf35, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:58:51,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-28T03:58:51,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 0dd90cc2721494b37accfc7c4d1bcf35, server=3a546af21488,36023,1732766230768 in 183 msec 2024-11-28T03:58:51,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-28T03:58:51,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=0dd90cc2721494b37accfc7c4d1bcf35, ASSIGN in 341 msec 2024-11-28T03:58:51,777 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:58:51,777 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766331777"}]},"ts":"1732766331777"} 2024-11-28T03:58:51,779 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-28T03:58:51,780 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:58:51,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1810 sec 2024-11-28T03:58:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-28T03:58:53,638 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-28T03:58:53,638 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-28T03:58:53,638 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:53,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-28T03:58:53,643 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:58:53,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-28T03:58:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:58:53,656 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:58:53,656 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-28T03:58:53,659 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T03:58:53,660 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-28T03:58:54,587 INFO [master/3a546af21488:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-28T03:58:54,587 INFO [master/3a546af21488:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-28T03:58:55,011 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-28T03:58:57,249 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:58:59,669 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35., hostname=3a546af21488,36023,1732766230768, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35., hostname=3a546af21488,36023,1732766230768, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a546af21488:36023 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-28T03:58:59,669 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35., hostname=3a546af21488,36023,1732766230768, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-28T03:58:59,669 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35., hostname=3a546af21488,36023,1732766230768, seqNum=2 from cache 2024-11-28T03:58:59,681 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=221 (was 231), ProcessCount=11 (was 11), AvailableMemoryMB=6506 (was 6527) 2024-11-28T03:58:59,691 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=221, ProcessCount=11, AvailableMemoryMB=6506 2024-11-28T03:58:59,694 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:58:59,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-28T03:58:59,699 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:58:59,699 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:58:59,699 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-28T03:58:59,700 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:58:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-28T03:58:59,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741851_1027 (size=407) 2024-11-28T03:58:59,713 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe84ed72aad8c4564ec88a7c73aca44b, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:58:59,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741852_1028 (size=58) 2024-11-28T03:58:59,724 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:58:59,724 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing fe84ed72aad8c4564ec88a7c73aca44b, disabling compactions & flushes 2024-11-28T03:58:59,724 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:58:59,725 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:58:59,725 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. after waiting 0 ms 2024-11-28T03:58:59,725 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:58:59,725 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:58:59,725 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe84ed72aad8c4564ec88a7c73aca44b: Waiting for close lock at 1732766339724Disabling compacts and flushes for region at 1732766339724Disabling writes for close at 1732766339725 (+1 ms)Writing region close event to WAL at 1732766339725Closed at 1732766339725 2024-11-28T03:58:59,727 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:58:59,728 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732766339727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766339727"}]},"ts":"1732766339727"} 2024-11-28T03:58:59,732 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:58:59,736 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:58:59,736 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766339736"}]},"ts":"1732766339736"} 2024-11-28T03:58:59,740 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-28T03:58:59,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fe84ed72aad8c4564ec88a7c73aca44b, ASSIGN}] 2024-11-28T03:58:59,742 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fe84ed72aad8c4564ec88a7c73aca44b, ASSIGN 2024-11-28T03:58:59,744 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fe84ed72aad8c4564ec88a7c73aca44b, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:58:59,895 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fe84ed72aad8c4564ec88a7c73aca44b, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:58:59,898 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fe84ed72aad8c4564ec88a7c73aca44b, ASSIGN because future has completed 2024-11-28T03:58:59,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe84ed72aad8c4564ec88a7c73aca44b, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:59:00,059 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:59:00,059 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => fe84ed72aad8c4564ec88a7c73aca44b, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:59:00,060 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,060 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:59:00,060 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,060 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,062 INFO [StoreOpener-fe84ed72aad8c4564ec88a7c73aca44b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,065 INFO [StoreOpener-fe84ed72aad8c4564ec88a7c73aca44b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe84ed72aad8c4564ec88a7c73aca44b columnFamilyName testFamily 2024-11-28T03:59:00,065 DEBUG [StoreOpener-fe84ed72aad8c4564ec88a7c73aca44b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:59:00,066 INFO [StoreOpener-fe84ed72aad8c4564ec88a7c73aca44b-1 {}] regionserver.HStore(327): Store=fe84ed72aad8c4564ec88a7c73aca44b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:59:00,066 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,067 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,068 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,069 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,069 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,072 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,076 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:59:00,077 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened fe84ed72aad8c4564ec88a7c73aca44b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69430265, jitterRate=0.03459157049655914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:59:00,077 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:00,078 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for fe84ed72aad8c4564ec88a7c73aca44b: Running coprocessor pre-open hook at 1732766340060Writing region info on filesystem at 1732766340060Initializing all the Stores at 1732766340062 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766340062Cleaning up temporary data from old regions at 1732766340069 (+7 ms)Running coprocessor post-open hooks at 1732766340077 (+8 ms)Region opened successfully at 1732766340078 (+1 ms) 2024-11-28T03:59:00,079 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b., pid=30, masterSystemTime=1732766340052 2024-11-28T03:59:00,084 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:59:00,084 INFO [RS_OPEN_PRIORITY_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:59:00,085 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=fe84ed72aad8c4564ec88a7c73aca44b, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:59:00,088 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe84ed72aad8c4564ec88a7c73aca44b, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:59:00,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-28T03:59:00,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure fe84ed72aad8c4564ec88a7c73aca44b, server=3a546af21488,36023,1732766230768 in 191 msec 2024-11-28T03:59:00,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-28T03:59:00,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=fe84ed72aad8c4564ec88a7c73aca44b, ASSIGN in 352 msec 2024-11-28T03:59:00,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:59:00,098 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766340097"}]},"ts":"1732766340097"} 2024-11-28T03:59:00,101 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-28T03:59:00,102 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:59:00,105 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 408 msec 2024-11-28T03:59:00,161 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-28T03:59:00,438 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-28T03:59:00,438 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T03:59:00,439 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-28T03:59:00,440 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-28T03:59:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-28T03:59:00,709 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-28T03:59:00,709 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-28T03:59:00,709 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:59:00,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-28T03:59:00,720 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:59:00,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-28T03:59:00,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:59:00,764 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:59:00,764 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-28T03:59:00,768 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-28T03:59:00,772 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-28T03:59:00,778 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-28T03:59:00,779 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-28T03:59:01,013 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-28T03:59:05,562 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:59:06,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36023 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283764' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-28T03:59:06,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36023 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:46454 deadline: 1732766352782, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283764' does not exist 2024-11-28T03:59:06,783 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 3a546af21488,36023,1732766230768 for closing scanner id = 6135864783863283764 for fe84ed72aad8c4564ec88a7c73aca44b of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6135864783863283764' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-28T03:59:06,789 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=255 (was 221) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6461 (was 6506) 2024-11-28T03:59:06,799 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=255, ProcessCount=11, AvailableMemoryMB=6461 2024-11-28T03:59:06,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-28T03:59:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-28T03:59:06,805 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-28T03:59:06,805 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:59:06,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-28T03:59:06,806 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-28T03:59:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T03:59:06,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741853_1029 (size=423) 2024-11-28T03:59:07,222 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb3d2634338b112f1ad83a0db23d04b2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd 2024-11-28T03:59:07,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741854_1030 (size=74) 2024-11-28T03:59:07,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:59:07,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing cb3d2634338b112f1ad83a0db23d04b2, disabling compactions & flushes 2024-11-28T03:59:07,230 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:07,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:07,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. after waiting 0 ms 2024-11-28T03:59:07,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:07,230 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:07,230 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb3d2634338b112f1ad83a0db23d04b2: Waiting for close lock at 1732766347230Disabling compacts and flushes for region at 1732766347230Disabling writes for close at 1732766347230Writing region close event to WAL at 1732766347230Closed at 1732766347230 2024-11-28T03:59:07,232 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-28T03:59:07,233 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732766347232"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732766347232"}]},"ts":"1732766347232"} 2024-11-28T03:59:07,235 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-28T03:59:07,237 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-28T03:59:07,237 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766347237"}]},"ts":"1732766347237"} 2024-11-28T03:59:07,239 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-28T03:59:07,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cb3d2634338b112f1ad83a0db23d04b2, ASSIGN}] 2024-11-28T03:59:07,241 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cb3d2634338b112f1ad83a0db23d04b2, ASSIGN 2024-11-28T03:59:07,242 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cb3d2634338b112f1ad83a0db23d04b2, ASSIGN; state=OFFLINE, location=3a546af21488,36023,1732766230768; forceNewPlan=false, retain=false 2024-11-28T03:59:07,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-28T03:59:07,392 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=cb3d2634338b112f1ad83a0db23d04b2, regionState=OPENING, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:59:07,396 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cb3d2634338b112f1ad83a0db23d04b2, ASSIGN because future has completed 2024-11-28T03:59:07,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb3d2634338b112f1ad83a0db23d04b2, server=3a546af21488,36023,1732766230768}] 2024-11-28T03:59:07,559 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:07,559 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => cb3d2634338b112f1ad83a0db23d04b2, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2.', STARTKEY => '', ENDKEY => ''} 2024-11-28T03:59:07,567 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,568 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-28T03:59:07,568 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,568 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,570 INFO [StoreOpener-cb3d2634338b112f1ad83a0db23d04b2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,572 INFO [StoreOpener-cb3d2634338b112f1ad83a0db23d04b2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb3d2634338b112f1ad83a0db23d04b2 columnFamilyName testFamily 2024-11-28T03:59:07,573 DEBUG [StoreOpener-cb3d2634338b112f1ad83a0db23d04b2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-28T03:59:07,573 INFO [StoreOpener-cb3d2634338b112f1ad83a0db23d04b2-1 {}] regionserver.HStore(327): Store=cb3d2634338b112f1ad83a0db23d04b2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-28T03:59:07,574 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,574 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,575 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,576 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,576 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,578 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,582 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-28T03:59:07,582 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened cb3d2634338b112f1ad83a0db23d04b2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70420632, jitterRate=0.049349188804626465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-28T03:59:07,583 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:07,583 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for cb3d2634338b112f1ad83a0db23d04b2: Running coprocessor pre-open hook at 1732766347568Writing region info on filesystem at 1732766347568Initializing all the Stores at 1732766347570 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732766347570Cleaning up temporary data from old regions at 1732766347576 (+6 ms)Running coprocessor post-open hooks at 1732766347583 (+7 ms)Region opened successfully at 1732766347583 2024-11-28T03:59:07,585 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., pid=33, masterSystemTime=1732766347548 2024-11-28T03:59:07,588 DEBUG [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:07,588 INFO [RS_OPEN_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:07,588 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=cb3d2634338b112f1ad83a0db23d04b2, regionState=OPEN, openSeqNum=2, regionLocation=3a546af21488,36023,1732766230768 2024-11-28T03:59:07,591 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb3d2634338b112f1ad83a0db23d04b2, server=3a546af21488,36023,1732766230768 because future has completed 2024-11-28T03:59:07,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-28T03:59:07,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure cb3d2634338b112f1ad83a0db23d04b2, server=3a546af21488,36023,1732766230768 in 196 msec 2024-11-28T03:59:07,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-28T03:59:07,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cb3d2634338b112f1ad83a0db23d04b2, ASSIGN in 355 msec 2024-11-28T03:59:07,598 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-28T03:59:07,599 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732766347598"}]},"ts":"1732766347598"} 2024-11-28T03:59:07,601 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-28T03:59:07,602 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-28T03:59:07,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 800 msec 2024-11-28T03:59:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45807 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-28T03:59:07,819 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-28T03:59:07,819 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-28T03:59:07,819 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:59:07,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-28T03:59:07,826 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-28T03:59:07,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-28T03:59:07,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:59:07,841 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-28T03:59:07,841 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-28T03:59:07,845 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-28T03:59:07,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T03:59:08,849 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a546af21488:36023 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-28T03:59:08,849 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-28T03:59:08,849 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2 from cache 2024-11-28T03:59:08,949 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-28T03:59:09,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-28T03:59:09,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2] 2024-11-28T03:59:09,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36023 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-28T03:59:10,438 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-28T03:59:10,438 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-28T03:59:10,849 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=3a546af21488:36023 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=989ms, rpcTimeout=984ms 2024-11-28T03:59:10,849 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=989ms, rpcTimeout=984ms 2024-11-28T03:59:10,849 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., hostname=3a546af21488,36023,1732766230768, seqNum=2 from cache 2024-11-28T03:59:10,860 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=267 (was 255) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6468 (was 6461) - AvailableMemoryMB LEAK? - 2024-11-28T03:59:10,861 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:59:10,861 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:59:10,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:59:10,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:59:10,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:59:10,869 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-28T03:59:10,869 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:59:10,869 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:59:10,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:59:10,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-28T03:59:10,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:59:10,870 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-28T03:59:10,870 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2048927636, stopped=false 2024-11-28T03:59:10,870 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3a546af21488,45807,1732766229951 2024-11-28T03:59:10,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T03:59:10,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-28T03:59:10,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:59:10,872 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:59:10,873 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T03:59:10,873 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-28T03:59:10,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T03:59:10,874 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:59:10,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:59:10,874 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3a546af21488,36023,1732766230768' ***** 2024-11-28T03:59:10,874 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-28T03:59:10,874 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-28T03:59:10,874 INFO [RS:0;3a546af21488:36023 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-28T03:59:10,875 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-28T03:59:10,875 INFO [RS:0;3a546af21488:36023 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-28T03:59:10,875 INFO [RS:0;3a546af21488:36023 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-28T03:59:10,875 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for 646cc9bf476050085baf95e2500a1dfc 2024-11-28T03:59:10,876 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for 0dd90cc2721494b37accfc7c4d1bcf35 2024-11-28T03:59:10,876 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for 5bf8ed37879fb968af0047760657b90a 2024-11-28T03:59:10,876 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for cb3d2634338b112f1ad83a0db23d04b2 2024-11-28T03:59:10,876 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for 7be12718b584945b0f2a51a2da19802a 2024-11-28T03:59:10,876 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for ea5f362764d0f80a2c40406d9057f965 2024-11-28T03:59:10,876 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for bd86fb12fcfed3489581169697519e39 2024-11-28T03:59:10,876 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:10,876 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:59:10,877 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for 4e7155b0e7763de9cbc461b8ae4b4836 2024-11-28T03:59:10,877 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(959): stopping server 3a546af21488,36023,1732766230768 2024-11-28T03:59:10,877 INFO [RS:0;3a546af21488:36023 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T03:59:10,877 INFO [RS:0;3a546af21488:36023 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3a546af21488:36023. 2024-11-28T03:59:10,877 DEBUG [RS:0;3a546af21488:36023 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-28T03:59:10,877 DEBUG [RS:0;3a546af21488:36023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:59:10,877 INFO [RS:0;3a546af21488:36023 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-28T03:59:10,877 INFO [RS:0;3a546af21488:36023 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-28T03:59:10,877 INFO [RS:0;3a546af21488:36023 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-28T03:59:10,877 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-28T03:59:10,878 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-28T03:59:10,878 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 646cc9bf476050085baf95e2500a1dfc, disabling compactions & flushes 2024-11-28T03:59:10,878 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-28T03:59:10,878 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 0dd90cc2721494b37accfc7c4d1bcf35, 1588230740, 4e7155b0e7763de9cbc461b8ae4b4836, 5bf8ed37879fb968af0047760657b90a, 646cc9bf476050085baf95e2500a1dfc, 7be12718b584945b0f2a51a2da19802a, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, cb3d2634338b112f1ad83a0db23d04b2, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:10,879 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-28T03:59:10,879 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:59:10,879 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-28T03:59:10,879 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:59:10,879 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. after waiting 0 ms 2024-11-28T03:59:10,879 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-28T03:59:10,879 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:59:10,879 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-28T03:59:10,883 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-28T03:59:10,883 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 646cc9bf476050085baf95e2500a1dfc 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:10,954 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc/.tmp/testFamily/47de1788d4494675ae86e0dfb7c24de1 is 61, key is row-0/testFamily:testQualifier/1732766327558/Put/seqid=0 2024-11-28T03:59:10,958 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/info/4eacd8adb99b40f2ba64475e3e8915ca is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836./info:regioninfo/1732766294760/Put/seqid=0 2024-11-28T03:59:10,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741855_1031 (size=20564) 2024-11-28T03:59:10,972 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/info/4eacd8adb99b40f2ba64475e3e8915ca 2024-11-28T03:59:10,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741856_1032 (size=5049) 2024-11-28T03:59:10,979 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc/.tmp/testFamily/47de1788d4494675ae86e0dfb7c24de1 2024-11-28T03:59:11,024 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc/.tmp/testFamily/47de1788d4494675ae86e0dfb7c24de1 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc/testFamily/47de1788d4494675ae86e0dfb7c24de1 2024-11-28T03:59:11,041 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc/testFamily/47de1788d4494675ae86e0dfb7c24de1, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:11,044 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/ns/61a31ad72fb841298ffbd574e9e8f644 is 43, key is default/ns:d/1732766293579/Put/seqid=0 2024-11-28T03:59:11,051 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 646cc9bf476050085baf95e2500a1dfc in 168ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:11,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741857_1033 (size=5153) 2024-11-28T03:59:11,065 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNext/646cc9bf476050085baf95e2500a1dfc/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:11,068 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:59:11,068 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 646cc9bf476050085baf95e2500a1dfc: Waiting for close lock at 1732766350878Running coprocessor pre-close hooks at 1732766350878Disabling compacts and flushes for region at 1732766350878Disabling writes for close at 1732766350879 (+1 ms)Obtaining lock to block concurrent updates at 1732766350883 (+4 ms)Preparing flush snapshotting stores in 646cc9bf476050085baf95e2500a1dfc at 1732766350883Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766350891 (+8 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. at 1732766350892 (+1 ms)Flushing 646cc9bf476050085baf95e2500a1dfc/testFamily: creating writer at 1732766350894 (+2 ms)Flushing 646cc9bf476050085baf95e2500a1dfc/testFamily: appending metadata at 1732766350947 (+53 ms)Flushing 646cc9bf476050085baf95e2500a1dfc/testFamily: closing flushed file at 1732766350951 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e8cb1ce: reopening flushed file at 1732766351021 (+70 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 646cc9bf476050085baf95e2500a1dfc in 168ms, sequenceid=8, compaction requested=false at 1732766351051 (+30 ms)Writing region close event to WAL at 1732766351059 (+8 ms)Running coprocessor post-close hooks at 1732766351066 (+7 ms)Closed at 1732766351068 (+2 ms) 2024-11-28T03:59:11,069 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732766324510.646cc9bf476050085baf95e2500a1dfc. 2024-11-28T03:59:11,069 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0dd90cc2721494b37accfc7c4d1bcf35, disabling compactions & flushes 2024-11-28T03:59:11,069 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:59:11,069 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:59:11,069 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. after waiting 0 ms 2024-11-28T03:59:11,069 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:59:11,069 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0dd90cc2721494b37accfc7c4d1bcf35 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:11,079 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 0dd90cc2721494b37accfc7c4d1bcf35, 1588230740, 4e7155b0e7763de9cbc461b8ae4b4836, 5bf8ed37879fb968af0047760657b90a, 7be12718b584945b0f2a51a2da19802a, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, cb3d2634338b112f1ad83a0db23d04b2, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:11,087 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35/.tmp/testFamily/be59c1f41d234151ae2ceb549651a270 is 61, key is row-0/testFamily:testQualifier/1732766333647/Put/seqid=0 2024-11-28T03:59:11,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741858_1034 (size=5049) 2024-11-28T03:59:11,093 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35/.tmp/testFamily/be59c1f41d234151ae2ceb549651a270 2024-11-28T03:59:11,102 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35/.tmp/testFamily/be59c1f41d234151ae2ceb549651a270 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35/testFamily/be59c1f41d234151ae2ceb549651a270 2024-11-28T03:59:11,110 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35/testFamily/be59c1f41d234151ae2ceb549651a270, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:11,111 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0dd90cc2721494b37accfc7c4d1bcf35 in 42ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:11,125 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/0dd90cc2721494b37accfc7c4d1bcf35/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:11,126 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:59:11,126 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0dd90cc2721494b37accfc7c4d1bcf35: Waiting for close lock at 1732766351069Running coprocessor pre-close hooks at 1732766351069Disabling compacts and flushes for region at 1732766351069Disabling writes for close at 1732766351069Obtaining lock to block concurrent updates at 1732766351069Preparing flush snapshotting stores in 0dd90cc2721494b37accfc7c4d1bcf35 at 1732766351069Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766351070 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. at 1732766351070Flushing 0dd90cc2721494b37accfc7c4d1bcf35/testFamily: creating writer at 1732766351070Flushing 0dd90cc2721494b37accfc7c4d1bcf35/testFamily: appending metadata at 1732766351087 (+17 ms)Flushing 0dd90cc2721494b37accfc7c4d1bcf35/testFamily: closing flushed file at 1732766351087Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cbe92e0: reopening flushed file at 1732766351101 (+14 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0dd90cc2721494b37accfc7c4d1bcf35 in 42ms, sequenceid=8, compaction requested=false at 1732766351111 (+10 ms)Writing region close event to WAL at 1732766351120 (+9 ms)Running coprocessor post-close hooks at 1732766351126 (+6 ms)Closed at 1732766351126 2024-11-28T03:59:11,126 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732766330598.0dd90cc2721494b37accfc7c4d1bcf35. 2024-11-28T03:59:11,126 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5bf8ed37879fb968af0047760657b90a, disabling compactions & flushes 2024-11-28T03:59:11,126 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:59:11,126 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:59:11,126 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. after waiting 0 ms 2024-11-28T03:59:11,126 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:59:11,126 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5bf8ed37879fb968af0047760657b90a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:11,144 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a/.tmp/testFamily/b09ad9056be948d3a535959936d66e81 is 61, key is row-0/testFamily:testQualifier/1732766312298/Put/seqid=0 2024-11-28T03:59:11,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741859_1035 (size=5049) 2024-11-28T03:59:11,151 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a/.tmp/testFamily/b09ad9056be948d3a535959936d66e81 2024-11-28T03:59:11,161 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a/.tmp/testFamily/b09ad9056be948d3a535959936d66e81 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a/testFamily/b09ad9056be948d3a535959936d66e81 2024-11-28T03:59:11,169 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a/testFamily/b09ad9056be948d3a535959936d66e81, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:11,170 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5bf8ed37879fb968af0047760657b90a in 44ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:11,176 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextException/5bf8ed37879fb968af0047760657b90a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:11,177 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:59:11,178 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5bf8ed37879fb968af0047760657b90a: Waiting for close lock at 1732766351126Running coprocessor pre-close hooks at 1732766351126Disabling compacts and flushes for region at 1732766351126Disabling writes for close at 1732766351126Obtaining lock to block concurrent updates at 1732766351126Preparing flush snapshotting stores in 5bf8ed37879fb968af0047760657b90a at 1732766351126Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766351127 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. at 1732766351127Flushing 5bf8ed37879fb968af0047760657b90a/testFamily: creating writer at 1732766351127Flushing 5bf8ed37879fb968af0047760657b90a/testFamily: appending metadata at 1732766351144 (+17 ms)Flushing 5bf8ed37879fb968af0047760657b90a/testFamily: closing flushed file at 1732766351144Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4afcc929: reopening flushed file at 1732766351160 (+16 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5bf8ed37879fb968af0047760657b90a in 44ms, sequenceid=8, compaction requested=false at 1732766351171 (+11 ms)Writing region close event to WAL at 1732766351171Running coprocessor post-close hooks at 1732766351177 (+6 ms)Closed at 1732766351177 2024-11-28T03:59:11,178 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732766309253.5bf8ed37879fb968af0047760657b90a. 2024-11-28T03:59:11,178 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cb3d2634338b112f1ad83a0db23d04b2, disabling compactions & flushes 2024-11-28T03:59:11,178 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:11,178 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:11,178 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. after waiting 0 ms 2024-11-28T03:59:11,178 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:11,178 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cb3d2634338b112f1ad83a0db23d04b2 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:11,203 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2/.tmp/testFamily/ca5b5ae88c7f4dfabfa028d248885dda is 61, key is row-0/testFamily:testQualifier/1732766347831/Put/seqid=0 2024-11-28T03:59:11,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741860_1036 (size=5049) 2024-11-28T03:59:11,279 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4e7155b0e7763de9cbc461b8ae4b4836, 7be12718b584945b0f2a51a2da19802a, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, cb3d2634338b112f1ad83a0db23d04b2, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:11,362 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-28T03:59:11,462 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/ns/61a31ad72fb841298ffbd574e9e8f644 2024-11-28T03:59:11,479 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4e7155b0e7763de9cbc461b8ae4b4836, 7be12718b584945b0f2a51a2da19802a, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, cb3d2634338b112f1ad83a0db23d04b2, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:11,492 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/table/9bb7d5eef0c44e99bc3960627d27f799 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732766294777/Put/seqid=0 2024-11-28T03:59:11,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741861_1037 (size=6699) 2024-11-28T03:59:11,610 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2/.tmp/testFamily/ca5b5ae88c7f4dfabfa028d248885dda 2024-11-28T03:59:11,619 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2/.tmp/testFamily/ca5b5ae88c7f4dfabfa028d248885dda as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2/testFamily/ca5b5ae88c7f4dfabfa028d248885dda 2024-11-28T03:59:11,627 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2/testFamily/ca5b5ae88c7f4dfabfa028d248885dda, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:11,628 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cb3d2634338b112f1ad83a0db23d04b2 in 450ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:11,628 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-28T03:59:11,633 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScannerAsync/cb3d2634338b112f1ad83a0db23d04b2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:11,634 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:11,634 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cb3d2634338b112f1ad83a0db23d04b2: Waiting for close lock at 1732766351178Running coprocessor pre-close hooks at 1732766351178Disabling compacts and flushes for region at 1732766351178Disabling writes for close at 1732766351178Obtaining lock to block concurrent updates at 1732766351178Preparing flush snapshotting stores in cb3d2634338b112f1ad83a0db23d04b2 at 1732766351178Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766351179 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. at 1732766351179Flushing cb3d2634338b112f1ad83a0db23d04b2/testFamily: creating writer at 1732766351179Flushing cb3d2634338b112f1ad83a0db23d04b2/testFamily: appending metadata at 1732766351203 (+24 ms)Flushing cb3d2634338b112f1ad83a0db23d04b2/testFamily: closing flushed file at 1732766351203Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31ed2a85: reopening flushed file at 1732766351618 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cb3d2634338b112f1ad83a0db23d04b2 in 450ms, sequenceid=8, compaction requested=false at 1732766351628 (+10 ms)Writing region close event to WAL at 1732766351629 (+1 ms)Running coprocessor post-close hooks at 1732766351634 (+5 ms)Closed at 1732766351634 2024-11-28T03:59:11,634 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732766346801.cb3d2634338b112f1ad83a0db23d04b2. 2024-11-28T03:59:11,635 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7be12718b584945b0f2a51a2da19802a, disabling compactions & flushes 2024-11-28T03:59:11,635 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:59:11,635 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:59:11,635 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. after waiting 0 ms 2024-11-28T03:59:11,635 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:59:11,635 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7be12718b584945b0f2a51a2da19802a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:11,652 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a/.tmp/testFamily/140584b2b7f1494783b855abf8e1c14f is 61, key is row-0/testFamily:testQualifier/1732766296100/Put/seqid=0 2024-11-28T03:59:11,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741862_1038 (size=5049) 2024-11-28T03:59:11,680 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4e7155b0e7763de9cbc461b8ae4b4836, 7be12718b584945b0f2a51a2da19802a, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:11,880 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1321): Waiting on 7 regions to close 2024-11-28T03:59:11,880 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7be12718b584945b0f2a51a2da19802a=hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a., ea5f362764d0f80a2c40406d9057f965=testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., bd86fb12fcfed3489581169697519e39=hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39., fe84ed72aad8c4564ec88a7c73aca44b=hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b., adba58a68e6b825d2651bbdf5f9778a4=testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4., 4e7155b0e7763de9cbc461b8ae4b4836=testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836.} 2024-11-28T03:59:11,880 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4e7155b0e7763de9cbc461b8ae4b4836, 7be12718b584945b0f2a51a2da19802a, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:11,881 INFO [regionserver/3a546af21488:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-28T03:59:11,881 INFO [regionserver/3a546af21488:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-28T03:59:11,899 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/table/9bb7d5eef0c44e99bc3960627d27f799 2024-11-28T03:59:11,910 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/info/4eacd8adb99b40f2ba64475e3e8915ca as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/info/4eacd8adb99b40f2ba64475e3e8915ca 2024-11-28T03:59:11,918 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/info/4eacd8adb99b40f2ba64475e3e8915ca, entries=100, sequenceid=56, filesize=20.1 K 2024-11-28T03:59:11,920 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/ns/61a31ad72fb841298ffbd574e9e8f644 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/ns/61a31ad72fb841298ffbd574e9e8f644 2024-11-28T03:59:11,929 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/ns/61a31ad72fb841298ffbd574e9e8f644, entries=2, sequenceid=56, filesize=5.0 K 2024-11-28T03:59:11,931 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/.tmp/table/9bb7d5eef0c44e99bc3960627d27f799 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/table/9bb7d5eef0c44e99bc3960627d27f799 2024-11-28T03:59:11,939 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/table/9bb7d5eef0c44e99bc3960627d27f799, entries=20, sequenceid=56, filesize=6.5 K 2024-11-28T03:59:11,940 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1061ms, sequenceid=56, compaction requested=false 2024-11-28T03:59:11,950 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-28T03:59:11,950 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-28T03:59:11,951 INFO [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-28T03:59:11,951 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732766350878Running coprocessor pre-close hooks at 1732766350878Disabling compacts and flushes for region at 1732766350878Disabling writes for close at 1732766350879 (+1 ms)Obtaining lock to block concurrent updates at 1732766350883 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1732766350883Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732766350891 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732766350892 (+1 ms)Flushing 1588230740/info: creating writer at 1732766350894 (+2 ms)Flushing 1588230740/info: appending metadata at 1732766350954 (+60 ms)Flushing 1588230740/info: closing flushed file at 1732766350954Flushing 1588230740/ns: creating writer at 1732766351021 (+67 ms)Flushing 1588230740/ns: appending metadata at 1732766351042 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732766351042Flushing 1588230740/table: creating writer at 1732766351472 (+430 ms)Flushing 1588230740/table: appending metadata at 1732766351492 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732766351492Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6068cc9d: reopening flushed file at 1732766351908 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22580ac4: reopening flushed file at 1732766351919 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b2ce5e2: reopening flushed file at 1732766351930 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1061ms, sequenceid=56, compaction requested=false at 1732766351940 (+10 ms)Writing region close event to WAL at 1732766351945 (+5 ms)Running coprocessor post-close hooks at 1732766351950 (+5 ms)Closed at 1732766351950 2024-11-28T03:59:11,951 DEBUG [RS_CLOSE_META-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-28T03:59:12,058 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a/.tmp/testFamily/140584b2b7f1494783b855abf8e1c14f 2024-11-28T03:59:12,068 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a/.tmp/testFamily/140584b2b7f1494783b855abf8e1c14f as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a/testFamily/140584b2b7f1494783b855abf8e1c14f 2024-11-28T03:59:12,077 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a/testFamily/140584b2b7f1494783b855abf8e1c14f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:12,079 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7be12718b584945b0f2a51a2da19802a in 444ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:12,080 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, 7be12718b584945b0f2a51a2da19802a, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:12,084 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnOpenScanner/7be12718b584945b0f2a51a2da19802a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:12,085 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:59:12,085 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7be12718b584945b0f2a51a2da19802a: Waiting for close lock at 1732766351634Running coprocessor pre-close hooks at 1732766351634Disabling compacts and flushes for region at 1732766351635 (+1 ms)Disabling writes for close at 1732766351635Obtaining lock to block concurrent updates at 1732766351635Preparing flush snapshotting stores in 7be12718b584945b0f2a51a2da19802a at 1732766351635Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766351635Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. at 1732766351636 (+1 ms)Flushing 7be12718b584945b0f2a51a2da19802a/testFamily: creating writer at 1732766351636Flushing 7be12718b584945b0f2a51a2da19802a/testFamily: appending metadata at 1732766351651 (+15 ms)Flushing 7be12718b584945b0f2a51a2da19802a/testFamily: closing flushed file at 1732766351651Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@746b8088: reopening flushed file at 1732766352067 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7be12718b584945b0f2a51a2da19802a in 444ms, sequenceid=8, compaction requested=false at 1732766352079 (+12 ms)Writing region close event to WAL at 1732766352080 (+1 ms)Running coprocessor post-close hooks at 1732766352085 (+5 ms)Closed at 1732766352085 2024-11-28T03:59:12,085 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732766295072.7be12718b584945b0f2a51a2da19802a. 2024-11-28T03:59:12,085 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ea5f362764d0f80a2c40406d9057f965, disabling compactions & flushes 2024-11-28T03:59:12,086 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:59:12,086 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:59:12,086 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. after waiting 0 ms 2024-11-28T03:59:12,086 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:59:12,086 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ea5f362764d0f80a2c40406d9057f965 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:12,111 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965/.tmp/testFamily/5d45a096dae44a33a61b2d3b0b5dc511 is 61, key is row-0/testFamily:testQualifier/1732766315388/Put/seqid=0 2024-11-28T03:59:12,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741863_1039 (size=5049) 2024-11-28T03:59:12,281 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:12,481 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, ea5f362764d0f80a2c40406d9057f965, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:12,518 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965/.tmp/testFamily/5d45a096dae44a33a61b2d3b0b5dc511 2024-11-28T03:59:12,528 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965/.tmp/testFamily/5d45a096dae44a33a61b2d3b0b5dc511 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965/testFamily/5d45a096dae44a33a61b2d3b0b5dc511 2024-11-28T03:59:12,537 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965/testFamily/5d45a096dae44a33a61b2d3b0b5dc511, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:12,538 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ea5f362764d0f80a2c40406d9057f965 in 452ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:12,544 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnOpenScanner/ea5f362764d0f80a2c40406d9057f965/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:12,545 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:59:12,545 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ea5f362764d0f80a2c40406d9057f965: Waiting for close lock at 1732766352085Running coprocessor pre-close hooks at 1732766352085Disabling compacts and flushes for region at 1732766352085Disabling writes for close at 1732766352086 (+1 ms)Obtaining lock to block concurrent updates at 1732766352086Preparing flush snapshotting stores in ea5f362764d0f80a2c40406d9057f965 at 1732766352086Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766352086Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. at 1732766352087 (+1 ms)Flushing ea5f362764d0f80a2c40406d9057f965/testFamily: creating writer at 1732766352087Flushing ea5f362764d0f80a2c40406d9057f965/testFamily: appending metadata at 1732766352110 (+23 ms)Flushing ea5f362764d0f80a2c40406d9057f965/testFamily: closing flushed file at 1732766352111 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34cb2c29: reopening flushed file at 1732766352527 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ea5f362764d0f80a2c40406d9057f965 in 452ms, sequenceid=8, compaction requested=false at 1732766352539 (+12 ms)Writing region close event to WAL at 1732766352539Running coprocessor post-close hooks at 1732766352545 (+6 ms)Closed at 1732766352545 2024-11-28T03:59:12,545 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732766312347.ea5f362764d0f80a2c40406d9057f965. 2024-11-28T03:59:12,545 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd86fb12fcfed3489581169697519e39, disabling compactions & flushes 2024-11-28T03:59:12,545 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:59:12,545 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:59:12,545 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. after waiting 0 ms 2024-11-28T03:59:12,545 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:59:12,545 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bd86fb12fcfed3489581169697519e39 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:12,563 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39/.tmp/testFamily/d63c118beffc4c789593ff2f5bd08000 is 61, key is row-0/testFamily:testQualifier/1732766303201/Put/seqid=0 2024-11-28T03:59:12,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741864_1040 (size=5049) 2024-11-28T03:59:12,681 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:12,881 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-28T03:59:12,882 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1325): Online Regions={bd86fb12fcfed3489581169697519e39=hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39., fe84ed72aad8c4564ec88a7c73aca44b=hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b., adba58a68e6b825d2651bbdf5f9778a4=testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4., 4e7155b0e7763de9cbc461b8ae4b4836=testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836.} 2024-11-28T03:59:12,882 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4, bd86fb12fcfed3489581169697519e39, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:12,969 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39/.tmp/testFamily/d63c118beffc4c789593ff2f5bd08000 2024-11-28T03:59:12,979 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39/.tmp/testFamily/d63c118beffc4c789593ff2f5bd08000 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39/testFamily/d63c118beffc4c789593ff2f5bd08000 2024-11-28T03:59:12,987 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39/testFamily/d63c118beffc4c789593ff2f5bd08000, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:12,988 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bd86fb12fcfed3489581169697519e39 in 443ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:12,993 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNextAsync/bd86fb12fcfed3489581169697519e39/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:12,994 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:59:12,994 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd86fb12fcfed3489581169697519e39: Waiting for close lock at 1732766352545Running coprocessor pre-close hooks at 1732766352545Disabling compacts and flushes for region at 1732766352545Disabling writes for close at 1732766352545Obtaining lock to block concurrent updates at 1732766352545Preparing flush snapshotting stores in bd86fb12fcfed3489581169697519e39 at 1732766352545Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766352546 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. at 1732766352546Flushing bd86fb12fcfed3489581169697519e39/testFamily: creating writer at 1732766352546Flushing bd86fb12fcfed3489581169697519e39/testFamily: appending metadata at 1732766352563 (+17 ms)Flushing bd86fb12fcfed3489581169697519e39/testFamily: closing flushed file at 1732766352563Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18fafcf8: reopening flushed file at 1732766352977 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for bd86fb12fcfed3489581169697519e39 in 443ms, sequenceid=8, compaction requested=false at 1732766352988 (+11 ms)Writing region close event to WAL at 1732766352989 (+1 ms)Running coprocessor post-close hooks at 1732766352994 (+5 ms)Closed at 1732766352994 2024-11-28T03:59:12,994 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732766302159.bd86fb12fcfed3489581169697519e39. 2024-11-28T03:59:12,994 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fe84ed72aad8c4564ec88a7c73aca44b, disabling compactions & flushes 2024-11-28T03:59:12,994 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:59:12,994 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:59:12,994 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. after waiting 0 ms 2024-11-28T03:59:12,994 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:59:12,994 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fe84ed72aad8c4564ec88a7c73aca44b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:13,013 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b/.tmp/testFamily/e0a87d308f754c9e8e628f8944180cf2 is 61, key is row-0/testFamily:testQualifier/1732766340739/Put/seqid=0 2024-11-28T03:59:13,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741865_1041 (size=5049) 2024-11-28T03:59:13,061 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-28T03:59:13,082 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:13,282 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4, fe84ed72aad8c4564ec88a7c73aca44b 2024-11-28T03:59:13,419 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b/.tmp/testFamily/e0a87d308f754c9e8e628f8944180cf2 2024-11-28T03:59:13,429 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b/.tmp/testFamily/e0a87d308f754c9e8e628f8944180cf2 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b/testFamily/e0a87d308f754c9e8e628f8944180cf2 2024-11-28T03:59:13,436 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b/testFamily/e0a87d308f754c9e8e628f8944180cf2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:13,438 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fe84ed72aad8c4564ec88a7c73aca44b in 444ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:13,443 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/hbase/testMetaScanTimeoutOnNext/fe84ed72aad8c4564ec88a7c73aca44b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:13,444 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:59:13,444 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fe84ed72aad8c4564ec88a7c73aca44b: Waiting for close lock at 1732766352994Running coprocessor pre-close hooks at 1732766352994Disabling compacts and flushes for region at 1732766352994Disabling writes for close at 1732766352994Obtaining lock to block concurrent updates at 1732766352994Preparing flush snapshotting stores in fe84ed72aad8c4564ec88a7c73aca44b at 1732766352995 (+1 ms)Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766352995Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. at 1732766352995Flushing fe84ed72aad8c4564ec88a7c73aca44b/testFamily: creating writer at 1732766352995Flushing fe84ed72aad8c4564ec88a7c73aca44b/testFamily: appending metadata at 1732766353012 (+17 ms)Flushing fe84ed72aad8c4564ec88a7c73aca44b/testFamily: closing flushed file at 1732766353012Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dc15c4a: reopening flushed file at 1732766353428 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for fe84ed72aad8c4564ec88a7c73aca44b in 444ms, sequenceid=8, compaction requested=false at 1732766353438 (+10 ms)Writing region close event to WAL at 1732766353439 (+1 ms)Running coprocessor post-close hooks at 1732766353444 (+5 ms)Closed at 1732766353444 2024-11-28T03:59:13,444 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732766339694.fe84ed72aad8c4564ec88a7c73aca44b. 2024-11-28T03:59:13,444 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing adba58a68e6b825d2651bbdf5f9778a4, disabling compactions & flushes 2024-11-28T03:59:13,445 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:59:13,445 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:59:13,445 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. after waiting 0 ms 2024-11-28T03:59:13,445 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:59:13,445 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing adba58a68e6b825d2651bbdf5f9778a4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:13,461 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4/.tmp/testFamily/35a88376554f4d178234f42319875286 is 61, key is row-0/testFamily:testQualifier/1732766321470/Put/seqid=0 2024-11-28T03:59:13,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741866_1042 (size=5049) 2024-11-28T03:59:13,483 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:59:13,683 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:59:13,717 DEBUG [master/3a546af21488:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-28T03:59:13,870 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4/.tmp/testFamily/35a88376554f4d178234f42319875286 2024-11-28T03:59:13,879 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4/.tmp/testFamily/35a88376554f4d178234f42319875286 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4/testFamily/35a88376554f4d178234f42319875286 2024-11-28T03:59:13,883 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-28T03:59:13,883 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1325): Online Regions={adba58a68e6b825d2651bbdf5f9778a4=testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4., 4e7155b0e7763de9cbc461b8ae4b4836=testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836.} 2024-11-28T03:59:13,883 DEBUG [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1351): Waiting on 4e7155b0e7763de9cbc461b8ae4b4836, adba58a68e6b825d2651bbdf5f9778a4 2024-11-28T03:59:13,887 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4/testFamily/35a88376554f4d178234f42319875286, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:13,888 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for adba58a68e6b825d2651bbdf5f9778a4 in 443ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:13,893 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testNormalScanTimeoutOnNextAsync/adba58a68e6b825d2651bbdf5f9778a4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:13,893 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:59:13,893 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for adba58a68e6b825d2651bbdf5f9778a4: Waiting for close lock at 1732766353444Running coprocessor pre-close hooks at 1732766353444Disabling compacts and flushes for region at 1732766353444Disabling writes for close at 1732766353445 (+1 ms)Obtaining lock to block concurrent updates at 1732766353445Preparing flush snapshotting stores in adba58a68e6b825d2651bbdf5f9778a4 at 1732766353445Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766353445Flushing stores of testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. at 1732766353446 (+1 ms)Flushing adba58a68e6b825d2651bbdf5f9778a4/testFamily: creating writer at 1732766353446Flushing adba58a68e6b825d2651bbdf5f9778a4/testFamily: appending metadata at 1732766353461 (+15 ms)Flushing adba58a68e6b825d2651bbdf5f9778a4/testFamily: closing flushed file at 1732766353461Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15399ac7: reopening flushed file at 1732766353878 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for adba58a68e6b825d2651bbdf5f9778a4 in 443ms, sequenceid=8, compaction requested=false at 1732766353888 (+10 ms)Writing region close event to WAL at 1732766353889 (+1 ms)Running coprocessor post-close hooks at 1732766353893 (+4 ms)Closed at 1732766353893 2024-11-28T03:59:13,894 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732766318428.adba58a68e6b825d2651bbdf5f9778a4. 2024-11-28T03:59:13,894 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4e7155b0e7763de9cbc461b8ae4b4836, disabling compactions & flushes 2024-11-28T03:59:13,894 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:59:13,894 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:59:13,894 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. after waiting 0 ms 2024-11-28T03:59:13,894 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:59:13,894 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4e7155b0e7763de9cbc461b8ae4b4836 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-28T03:59:13,919 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836/.tmp/testFamily/51ccd770e56d44c1aaab1c1f4baff886 is 61, key is row-0/testFamily:testQualifier/1732766294985/Put/seqid=0 2024-11-28T03:59:13,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741867_1043 (size=5049) 2024-11-28T03:59:13,926 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836/.tmp/testFamily/51ccd770e56d44c1aaab1c1f4baff886 2024-11-28T03:59:13,935 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836/.tmp/testFamily/51ccd770e56d44c1aaab1c1f4baff886 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836/testFamily/51ccd770e56d44c1aaab1c1f4baff886 2024-11-28T03:59:13,944 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836/testFamily/51ccd770e56d44c1aaab1c1f4baff886, entries=4, sequenceid=8, filesize=4.9 K 2024-11-28T03:59:13,945 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4e7155b0e7763de9cbc461b8ae4b4836 in 51ms, sequenceid=8, compaction requested=false 2024-11-28T03:59:13,950 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/data/default/testRetryOutOfOrderScannerNextExceptionAsync/4e7155b0e7763de9cbc461b8ae4b4836/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-28T03:59:13,951 INFO [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:59:13,951 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4e7155b0e7763de9cbc461b8ae4b4836: Waiting for close lock at 1732766353894Running coprocessor pre-close hooks at 1732766353894Disabling compacts and flushes for region at 1732766353894Disabling writes for close at 1732766353894Obtaining lock to block concurrent updates at 1732766353894Preparing flush snapshotting stores in 4e7155b0e7763de9cbc461b8ae4b4836 at 1732766353894Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732766353894Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. at 1732766353895 (+1 ms)Flushing 4e7155b0e7763de9cbc461b8ae4b4836/testFamily: creating writer at 1732766353895Flushing 4e7155b0e7763de9cbc461b8ae4b4836/testFamily: appending metadata at 1732766353919 (+24 ms)Flushing 4e7155b0e7763de9cbc461b8ae4b4836/testFamily: closing flushed file at 1732766353919Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35d8fb83: reopening flushed file at 1732766353934 (+15 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4e7155b0e7763de9cbc461b8ae4b4836 in 51ms, sequenceid=8, compaction requested=false at 1732766353945 (+11 ms)Writing region close event to WAL at 1732766353946 (+1 ms)Running coprocessor post-close hooks at 1732766353951 (+5 ms)Closed at 1732766353951 2024-11-28T03:59:13,951 DEBUG [RS_CLOSE_REGION-regionserver/3a546af21488:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732766293912.4e7155b0e7763de9cbc461b8ae4b4836. 2024-11-28T03:59:14,084 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(976): stopping server 3a546af21488,36023,1732766230768; all regions closed. 2024-11-28T03:59:14,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741834_1010 (size=19479) 2024-11-28T03:59:14,095 DEBUG [RS:0;3a546af21488:36023 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/oldWALs 2024-11-28T03:59:14,095 INFO [RS:0;3a546af21488:36023 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3a546af21488%2C36023%2C1732766230768.meta:.meta(num 1732766293388) 2024-11-28T03:59:14,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741833_1009 (size=20031) 2024-11-28T03:59:14,101 DEBUG [RS:0;3a546af21488:36023 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/oldWALs 2024-11-28T03:59:14,101 INFO [RS:0;3a546af21488:36023 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3a546af21488%2C36023%2C1732766230768:(num 1732766292858) 2024-11-28T03:59:14,101 DEBUG [RS:0;3a546af21488:36023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-28T03:59:14,101 INFO [RS:0;3a546af21488:36023 {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T03:59:14,101 INFO [RS:0;3a546af21488:36023 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T03:59:14,101 INFO [RS:0;3a546af21488:36023 {}] hbase.ChoreService(370): Chore service for: regionserver/3a546af21488:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-28T03:59:14,101 INFO [RS:0;3a546af21488:36023 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T03:59:14,101 INFO [regionserver/3a546af21488:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T03:59:14,103 INFO [RS:0;3a546af21488:36023 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36023 2024-11-28T03:59:14,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3a546af21488,36023,1732766230768 2024-11-28T03:59:14,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-28T03:59:14,107 INFO [RS:0;3a546af21488:36023 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T03:59:14,109 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3a546af21488,36023,1732766230768] 2024-11-28T03:59:14,111 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3a546af21488,36023,1732766230768 already deleted, retry=false 2024-11-28T03:59:14,111 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3a546af21488,36023,1732766230768 expired; onlineServers=0 2024-11-28T03:59:14,111 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3a546af21488,45807,1732766229951' ***** 2024-11-28T03:59:14,112 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-28T03:59:14,112 INFO [M:0;3a546af21488:45807 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-28T03:59:14,112 INFO [M:0;3a546af21488:45807 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-28T03:59:14,112 DEBUG [M:0;3a546af21488:45807 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-28T03:59:14,112 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-28T03:59:14,112 DEBUG [M:0;3a546af21488:45807 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-28T03:59:14,112 DEBUG [master/3a546af21488:0:becomeActiveMaster-HFileCleaner.small.0-1732766232733 {}] cleaner.HFileCleaner(306): Exit Thread[master/3a546af21488:0:becomeActiveMaster-HFileCleaner.small.0-1732766232733,5,FailOnTimeoutGroup] 2024-11-28T03:59:14,112 DEBUG [master/3a546af21488:0:becomeActiveMaster-HFileCleaner.large.0-1732766232732 {}] cleaner.HFileCleaner(306): Exit Thread[master/3a546af21488:0:becomeActiveMaster-HFileCleaner.large.0-1732766232732,5,FailOnTimeoutGroup] 2024-11-28T03:59:14,112 INFO [M:0;3a546af21488:45807 {}] hbase.ChoreService(370): Chore service for: master/3a546af21488:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-28T03:59:14,112 INFO [M:0;3a546af21488:45807 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-28T03:59:14,113 DEBUG [M:0;3a546af21488:45807 {}] master.HMaster(1795): Stopping service threads 2024-11-28T03:59:14,113 INFO [M:0;3a546af21488:45807 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-28T03:59:14,113 INFO [M:0;3a546af21488:45807 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-28T03:59:14,114 INFO [M:0;3a546af21488:45807 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-28T03:59:14,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-28T03:59:14,114 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-28T03:59:14,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-28T03:59:14,114 DEBUG [M:0;3a546af21488:45807 {}] zookeeper.ZKUtil(347): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-28T03:59:14,115 WARN [M:0;3a546af21488:45807 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-28T03:59:14,115 INFO [M:0;3a546af21488:45807 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/.lastflushedseqids 2024-11-28T03:59:14,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741868_1044 (size=460) 2024-11-28T03:59:14,130 INFO [M:0;3a546af21488:45807 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-28T03:59:14,130 INFO [M:0;3a546af21488:45807 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-28T03:59:14,131 DEBUG [M:0;3a546af21488:45807 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-28T03:59:14,131 INFO [M:0;3a546af21488:45807 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:59:14,131 DEBUG [M:0;3a546af21488:45807 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:59:14,131 DEBUG [M:0;3a546af21488:45807 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-28T03:59:14,131 DEBUG [M:0;3a546af21488:45807 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:59:14,131 INFO [M:0;3a546af21488:45807 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.78 KB heapSize=190.25 KB 2024-11-28T03:59:14,156 DEBUG [M:0;3a546af21488:45807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bf3e6f9375d548a29956c6e617e324ca is 82, key is hbase:meta,,1/info:regioninfo/1732766293495/Put/seqid=0 2024-11-28T03:59:14,171 INFO [regionserver/3a546af21488:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-28T03:59:14,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741869_1045 (size=5672) 2024-11-28T03:59:14,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T03:59:14,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36023-0x1003c461aff0001, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T03:59:14,210 INFO [RS:0;3a546af21488:36023 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T03:59:14,210 INFO [RS:0;3a546af21488:36023 {}] regionserver.HRegionServer(1031): Exiting; stopping=3a546af21488,36023,1732766230768; zookeeper connection closed. 2024-11-28T03:59:14,211 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5c62ecd4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5c62ecd4 2024-11-28T03:59:14,211 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-28T03:59:14,573 INFO [M:0;3a546af21488:45807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bf3e6f9375d548a29956c6e617e324ca 2024-11-28T03:59:14,604 DEBUG [M:0;3a546af21488:45807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fcf079e8ac064eafaecc4fccb73a6091 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732766294785/Put/seqid=0 2024-11-28T03:59:14,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741870_1046 (size=14068) 2024-11-28T03:59:15,014 INFO [M:0;3a546af21488:45807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.22 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fcf079e8ac064eafaecc4fccb73a6091 2024-11-28T03:59:15,022 INFO [M:0;3a546af21488:45807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fcf079e8ac064eafaecc4fccb73a6091 2024-11-28T03:59:15,049 DEBUG [M:0;3a546af21488:45807 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c29969400e364a029bb2b40c5c707025 is 69, key is 3a546af21488,36023,1732766230768/rs:state/1732766262723/Put/seqid=0 2024-11-28T03:59:15,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741871_1047 (size=5156) 2024-11-28T03:59:15,457 INFO [M:0;3a546af21488:45807 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c29969400e364a029bb2b40c5c707025 2024-11-28T03:59:15,465 DEBUG [M:0;3a546af21488:45807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bf3e6f9375d548a29956c6e617e324ca as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bf3e6f9375d548a29956c6e617e324ca 2024-11-28T03:59:15,473 INFO [M:0;3a546af21488:45807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bf3e6f9375d548a29956c6e617e324ca, entries=8, sequenceid=311, filesize=5.5 K 2024-11-28T03:59:15,475 DEBUG [M:0;3a546af21488:45807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fcf079e8ac064eafaecc4fccb73a6091 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fcf079e8ac064eafaecc4fccb73a6091 2024-11-28T03:59:15,483 INFO [M:0;3a546af21488:45807 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fcf079e8ac064eafaecc4fccb73a6091 2024-11-28T03:59:15,483 INFO [M:0;3a546af21488:45807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fcf079e8ac064eafaecc4fccb73a6091, entries=33, sequenceid=311, filesize=13.7 K 2024-11-28T03:59:15,485 DEBUG [M:0;3a546af21488:45807 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c29969400e364a029bb2b40c5c707025 as hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c29969400e364a029bb2b40c5c707025 2024-11-28T03:59:15,492 INFO [M:0;3a546af21488:45807 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46373/user/jenkins/test-data/e7593eaf-fa86-1039-b367-45cb091c2abd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c29969400e364a029bb2b40c5c707025, entries=1, sequenceid=311, filesize=5.0 K 2024-11-28T03:59:15,493 INFO [M:0;3a546af21488:45807 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.78 KB/166687, heapSize ~189.95 KB/194512, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1362ms, sequenceid=311, compaction requested=false 2024-11-28T03:59:15,495 INFO [M:0;3a546af21488:45807 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-28T03:59:15,495 DEBUG [M:0;3a546af21488:45807 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732766354130Disabling compacts and flushes for region at 1732766354130Disabling writes for close at 1732766354131 (+1 ms)Obtaining lock to block concurrent updates at 1732766354131Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732766354131Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166687, getHeapSize=194752, getOffHeapSize=0, getCellsCount=358 at 1732766354132 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732766354132Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732766354132Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732766354156 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732766354156Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732766354581 (+425 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732766354603 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732766354603Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732766355022 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732766355048 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732766355048Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e1e4493: reopening flushed file at 1732766355464 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10c27b2: reopening flushed file at 1732766355474 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f52f43b: reopening flushed file at 1732766355483 (+9 ms)Finished flush of dataSize ~162.78 KB/166687, heapSize ~189.95 KB/194512, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1362ms, sequenceid=311, compaction requested=false at 1732766355493 (+10 ms)Writing region close event to WAL at 1732766355495 (+2 ms)Closed at 1732766355495 2024-11-28T03:59:15,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39619 is added to blk_1073741830_1006 (size=187509) 2024-11-28T03:59:15,499 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-28T03:59:15,499 INFO [M:0;3a546af21488:45807 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-28T03:59:15,499 INFO [M:0;3a546af21488:45807 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45807 2024-11-28T03:59:15,500 INFO [M:0;3a546af21488:45807 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-28T03:59:15,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T03:59:15,602 INFO [M:0;3a546af21488:45807 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-28T03:59:15,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45807-0x1003c461aff0000, quorum=127.0.0.1:64005, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-28T03:59:15,607 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7722c06d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-28T03:59:15,610 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@445877d1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T03:59:15,610 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T03:59:15,611 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@23178064{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T03:59:15,611 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@662cf64e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/hadoop.log.dir/,STOPPED} 2024-11-28T03:59:15,616 WARN [BP-1691337829-172.17.0.2-1732766226774 heartbeating to localhost/127.0.0.1:46373 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-28T03:59:15,616 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-28T03:59:15,616 WARN [BP-1691337829-172.17.0.2-1732766226774 heartbeating to localhost/127.0.0.1:46373 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1691337829-172.17.0.2-1732766226774 (Datanode Uuid da0b1283-7ca7-4081-83e0-b9a6b833dbdb) service to localhost/127.0.0.1:46373 2024-11-28T03:59:15,616 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-28T03:59:15,617 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/cluster_7e20e934-fbf9-def3-20eb-b2cc66e44915/data/data1/current/BP-1691337829-172.17.0.2-1732766226774 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T03:59:15,617 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/cluster_7e20e934-fbf9-def3-20eb-b2cc66e44915/data/data2/current/BP-1691337829-172.17.0.2-1732766226774 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-28T03:59:15,618 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-28T03:59:15,629 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-28T03:59:15,630 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-28T03:59:15,630 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-28T03:59:15,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-28T03:59:15,630 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e8cd362-63d5-d7ca-fd2d-59390fb8ee64/hadoop.log.dir/,STOPPED} 2024-11-28T03:59:15,641 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-28T03:59:15,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down