2024-11-25 03:38:22,413 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-25 03:38:22,432 main DEBUG Took 0.009664 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 03:38:22,432 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 03:38:22,432 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 03:38:22,433 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 03:38:22,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,441 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 03:38:22,453 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,455 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,455 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,456 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,457 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,458 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,459 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,460 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,461 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,461 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,461 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,462 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,463 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:38:22,463 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,464 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 03:38:22,465 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:38:22,466 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 03:38:22,468 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 03:38:22,468 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 03:38:22,469 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 03:38:22,470 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 03:38:22,478 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 03:38:22,481 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 03:38:22,483 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 03:38:22,483 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 03:38:22,483 main DEBUG createAppenders(={Console}) 2024-11-25 03:38:22,484 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-25 03:38:22,484 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-25 03:38:22,485 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-25 03:38:22,486 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 03:38:22,486 main DEBUG OutputStream closed 2024-11-25 03:38:22,486 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 03:38:22,487 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 03:38:22,487 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-25 03:38:22,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 03:38:22,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 03:38:22,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 03:38:22,556 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 03:38:22,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 03:38:22,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 03:38:22,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 03:38:22,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 03:38:22,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 03:38:22,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 03:38:22,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 03:38:22,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 03:38:22,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 03:38:22,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 03:38:22,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 03:38:22,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 03:38:22,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 03:38:22,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 03:38:22,563 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 03:38:22,564 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-25 03:38:22,564 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 03:38:22,565 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-25T03:38:22,580 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-25 03:38:22,583 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 03:38:22,583 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T03:38:22,853 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318 2024-11-25T03:38:22,870 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T03:38:22,890 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/cluster_052b3b17-9ee8-7e2f-08fb-e0d632542370, deleteOnExit=true 2024-11-25T03:38:22,891 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T03:38:22,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/test.cache.data in system properties and HBase conf 2024-11-25T03:38:22,893 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T03:38:22,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/hadoop.log.dir in system properties and HBase conf 2024-11-25T03:38:22,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T03:38:22,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T03:38:22,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T03:38:22,991 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T03:38:23,085 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:38:23,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:38:23,089 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:38:23,090 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T03:38:23,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:38:23,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T03:38:23,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T03:38:23,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:38:23,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:38:23,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T03:38:23,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/nfs.dump.dir in system properties and HBase conf 2024-11-25T03:38:23,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/java.io.tmpdir in system properties and HBase conf 2024-11-25T03:38:23,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:38:23,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T03:38:23,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T03:38:23,932 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T03:38:24,013 INFO [Time-limited test {}] log.Log(170): Logging initialized @2266ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T03:38:24,087 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:38:24,145 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:38:24,164 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:38:24,165 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:38:24,166 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:38:24,178 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:38:24,180 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:38:24,181 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:38:24,377 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/java.io.tmpdir/jetty-localhost-42465-hadoop-hdfs-3_4_1-tests_jar-_-any-18324805369094195899/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:38:24,383 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:42465} 2024-11-25T03:38:24,384 INFO [Time-limited test {}] server.Server(415): Started @2638ms 2024-11-25T03:38:24,777 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:38:24,784 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:38:24,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:38:24,786 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:38:24,786 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T03:38:24,787 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:38:24,788 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:38:24,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@429b6cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/java.io.tmpdir/jetty-localhost-45581-hadoop-hdfs-3_4_1-tests_jar-_-any-8447983436267289448/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:38:24,911 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:45581} 2024-11-25T03:38:24,911 INFO [Time-limited test {}] server.Server(415): Started @3166ms 2024-11-25T03:38:24,967 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:38:25,443 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/cluster_052b3b17-9ee8-7e2f-08fb-e0d632542370/data/data1/current/BP-493315070-172.17.0.2-1732505903679/current, will proceed with Du for space computation calculation, 2024-11-25T03:38:25,443 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/cluster_052b3b17-9ee8-7e2f-08fb-e0d632542370/data/data2/current/BP-493315070-172.17.0.2-1732505903679/current, will proceed with Du for space computation calculation, 2024-11-25T03:38:25,485 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:38:25,550 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc77b3c14cd350503 with lease ID 0x6604f57df587b13: Processing first storage report for DS-b1fb3875-2822-4fc5-9221-02816a87253b from datanode DatanodeRegistration(127.0.0.1:42577, datanodeUuid=5a8be03a-2f9b-4613-8646-d792372e733b, infoPort=34425, infoSecurePort=0, ipcPort=46853, storageInfo=lv=-57;cid=testClusterID;nsid=2044374662;c=1732505903679) 2024-11-25T03:38:25,552 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc77b3c14cd350503 with lease ID 0x6604f57df587b13: from storage DS-b1fb3875-2822-4fc5-9221-02816a87253b node DatanodeRegistration(127.0.0.1:42577, datanodeUuid=5a8be03a-2f9b-4613-8646-d792372e733b, infoPort=34425, infoSecurePort=0, ipcPort=46853, storageInfo=lv=-57;cid=testClusterID;nsid=2044374662;c=1732505903679), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-25T03:38:25,553 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc77b3c14cd350503 with lease ID 0x6604f57df587b13: Processing first storage report for DS-84a949ad-59f3-4e9d-b1b4-87aa7d960a58 from datanode DatanodeRegistration(127.0.0.1:42577, datanodeUuid=5a8be03a-2f9b-4613-8646-d792372e733b, infoPort=34425, infoSecurePort=0, ipcPort=46853, storageInfo=lv=-57;cid=testClusterID;nsid=2044374662;c=1732505903679) 2024-11-25T03:38:25,553 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc77b3c14cd350503 with lease ID 0x6604f57df587b13: from storage DS-84a949ad-59f3-4e9d-b1b4-87aa7d960a58 node DatanodeRegistration(127.0.0.1:42577, datanodeUuid=5a8be03a-2f9b-4613-8646-d792372e733b, infoPort=34425, infoSecurePort=0, ipcPort=46853, storageInfo=lv=-57;cid=testClusterID;nsid=2044374662;c=1732505903679), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:38:25,617 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318 2024-11-25T03:38:25,722 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/cluster_052b3b17-9ee8-7e2f-08fb-e0d632542370/zookeeper_0, clientPort=58447, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/cluster_052b3b17-9ee8-7e2f-08fb-e0d632542370/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/cluster_052b3b17-9ee8-7e2f-08fb-e0d632542370/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T03:38:25,732 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58447 2024-11-25T03:38:25,744 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:38:25,747 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:38:26,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:38:26,449 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 with version=8 2024-11-25T03:38:26,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/hbase-staging 2024-11-25T03:38:26,567 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T03:38:26,890 INFO [Time-limited test {}] client.ConnectionUtils(128): master/38a5d2077268:0 server-side Connection retries=9 2024-11-25T03:38:26,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:38:26,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:38:26,908 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:38:26,908 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:38:26,908 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:38:27,082 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T03:38:27,157 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T03:38:27,167 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T03:38:27,171 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:38:27,200 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21805 (auto-detected) 2024-11-25T03:38:27,202 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T03:38:27,241 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46447 2024-11-25T03:38:27,277 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46447 connecting to ZooKeeper ensemble=127.0.0.1:58447 2024-11-25T03:38:27,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464470x0, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:38:27,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46447-0x1014d3414b00000 connected 2024-11-25T03:38:27,370 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:38:27,375 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:38:27,390 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:38:27,395 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977, hbase.cluster.distributed=false 2024-11-25T03:38:27,437 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:38:27,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46447 2024-11-25T03:38:27,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46447 2024-11-25T03:38:27,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46447 2024-11-25T03:38:27,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46447 2024-11-25T03:38:27,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46447 2024-11-25T03:38:27,656 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/38a5d2077268:0 server-side Connection retries=9 2024-11-25T03:38:27,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:38:27,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:38:27,659 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:38:27,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:38:27,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:38:27,664 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:38:27,667 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:38:27,668 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36415 2024-11-25T03:38:27,671 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36415 connecting to ZooKeeper ensemble=127.0.0.1:58447 2024-11-25T03:38:27,673 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:38:27,678 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:38:27,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364150x0, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:38:27,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:364150x0, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:38:27,697 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:38:27,705 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36415-0x1014d3414b00001 connected 2024-11-25T03:38:27,712 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:38:27,715 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:38:27,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:38:27,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36415 2024-11-25T03:38:27,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36415 2024-11-25T03:38:27,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36415 2024-11-25T03:38:27,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36415 2024-11-25T03:38:27,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36415 2024-11-25T03:38:27,761 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;38a5d2077268:46447 2024-11-25T03:38:27,764 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/38a5d2077268,46447,1732505906635 2024-11-25T03:38:27,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:38:27,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:38:27,773 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/38a5d2077268,46447,1732505906635 2024-11-25T03:38:27,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:27,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:38:27,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:27,806 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:38:27,807 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/38a5d2077268,46447,1732505906635 from backup master directory 2024-11-25T03:38:27,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/38a5d2077268,46447,1732505906635 2024-11-25T03:38:27,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:38:27,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:38:27,813 WARN [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:38:27,813 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=38a5d2077268,46447,1732505906635 2024-11-25T03:38:27,816 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T03:38:27,817 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T03:38:27,882 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/hbase.id] with ID: 68fa995c-d5c8-49e7-9647-a8edabc4bcd7 2024-11-25T03:38:27,883 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/.tmp/hbase.id 2024-11-25T03:38:27,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:38:27,896 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/.tmp/hbase.id]:[hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/hbase.id] 2024-11-25T03:38:27,947 INFO [master/38a5d2077268:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:38:27,952 INFO [master/38a5d2077268:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T03:38:27,974 INFO [master/38a5d2077268:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-25T03:38:27,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:27,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:27,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:38:28,421 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:38:28,424 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T03:38:28,442 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:38:28,447 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:38:28,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:38:28,909 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store 2024-11-25T03:38:28,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:38:29,342 INFO [master/38a5d2077268:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T03:38:29,346 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:38:29,348 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:38:29,348 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:38:29,348 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:38:29,350 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:38:29,350 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:38:29,350 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:38:29,352 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732505909347Disabling compacts and flushes for region at 1732505909347Disabling writes for close at 1732505909350 (+3 ms)Writing region close event to WAL at 1732505909350Closed at 1732505909350 2024-11-25T03:38:29,355 WARN [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/.initializing 2024-11-25T03:38:29,355 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/WALs/38a5d2077268,46447,1732505906635 2024-11-25T03:38:29,365 INFO [master/38a5d2077268:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:38:29,384 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C46447%2C1732505906635, suffix=, logDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/WALs/38a5d2077268,46447,1732505906635, archiveDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/oldWALs, maxLogs=10 2024-11-25T03:38:29,414 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/WALs/38a5d2077268,46447,1732505906635/38a5d2077268%2C46447%2C1732505906635.1732505909390, exclude list is [], retry=0 2024-11-25T03:38:29,436 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42577,DS-b1fb3875-2822-4fc5-9221-02816a87253b,DISK] 2024-11-25T03:38:29,440 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T03:38:29,493 INFO [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/WALs/38a5d2077268,46447,1732505906635/38a5d2077268%2C46447%2C1732505906635.1732505909390 2024-11-25T03:38:29,494 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34425:34425)] 2024-11-25T03:38:29,494 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:38:29,495 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:38:29,498 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,499 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,543 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,576 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T03:38:29,580 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:29,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:38:29,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,590 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T03:38:29,590 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:29,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:29,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,594 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T03:38:29,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:29,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:29,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T03:38:29,600 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:29,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:38:29,602 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,607 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,608 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,615 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,616 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,619 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:38:29,626 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:38:29,631 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:38:29,633 INFO [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62892478, jitterRate=-0.06282904744148254}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:38:29,641 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732505909513Initializing all the Stores at 1732505909516 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505909516Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505909517 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505909518 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505909518Cleaning up temporary data from old regions at 1732505909616 (+98 ms)Region opened successfully at 1732505909640 (+24 ms) 2024-11-25T03:38:29,642 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T03:38:29,682 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600cd222, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=38a5d2077268/172.17.0.2:0 2024-11-25T03:38:29,718 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T03:38:29,731 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T03:38:29,731 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T03:38:29,734 INFO [master/38a5d2077268:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:38:29,736 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T03:38:29,741 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-25T03:38:29,741 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T03:38:29,776 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T03:38:29,787 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T03:38:29,789 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T03:38:29,792 INFO [master/38a5d2077268:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T03:38:29,794 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T03:38:29,795 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T03:38:29,798 INFO [master/38a5d2077268:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T03:38:29,802 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T03:38:29,804 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T03:38:29,806 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T03:38:29,811 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T03:38:29,834 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T03:38:29,836 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T03:38:29,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:38:29,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:29,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:38:29,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:29,847 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=38a5d2077268,46447,1732505906635, sessionid=0x1014d3414b00000, setting cluster-up flag (Was=false) 2024-11-25T03:38:29,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:29,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:29,873 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T03:38:29,879 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=38a5d2077268,46447,1732505906635 2024-11-25T03:38:29,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:29,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:38:29,899 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T03:38:29,901 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=38a5d2077268,46447,1732505906635 2024-11-25T03:38:29,910 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T03:38:29,947 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(746): ClusterId : 68fa995c-d5c8-49e7-9647-a8edabc4bcd7 2024-11-25T03:38:29,950 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:38:29,956 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:38:29,956 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:38:29,962 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:38:29,963 DEBUG [RS:0;38a5d2077268:36415 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52d1332f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=38a5d2077268/172.17.0.2:0 2024-11-25T03:38:29,987 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;38a5d2077268:36415 2024-11-25T03:38:29,991 INFO [RS:0;38a5d2077268:36415 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:38:29,991 INFO [RS:0;38a5d2077268:36415 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:38:29,991 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:38:29,994 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(2659): reportForDuty to master=38a5d2077268,46447,1732505906635 with port=36415, startcode=1732505907601 2024-11-25T03:38:30,009 DEBUG [RS:0;38a5d2077268:36415 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:38:30,016 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:38:30,035 INFO [master/38a5d2077268:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T03:38:30,047 INFO [master/38a5d2077268:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T03:38:30,055 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 38a5d2077268,46447,1732505906635 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T03:38:30,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58607, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:38:30,070 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46447 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:38:30,075 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:38:30,075 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:38:30,075 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:38:30,078 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/38a5d2077268:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:38:30,078 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/38a5d2077268:0, corePoolSize=10, maxPoolSize=10 2024-11-25T03:38:30,078 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:38:30,079 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/38a5d2077268:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:38:30,079 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:38:30,090 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732505940090 2024-11-25T03:38:30,092 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T03:38:30,093 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:38:30,093 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T03:38:30,094 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T03:38:30,098 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T03:38:30,098 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T03:38:30,099 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T03:38:30,099 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T03:38:30,102 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:30,102 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T03:38:30,107 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-25T03:38:30,107 WARN [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-25T03:38:30,107 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:38:30,117 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T03:38:30,118 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T03:38:30,119 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T03:38:30,122 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T03:38:30,123 INFO [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T03:38:30,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:38:30,128 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732505910124,5,FailOnTimeoutGroup] 2024-11-25T03:38:30,132 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732505910129,5,FailOnTimeoutGroup] 2024-11-25T03:38:30,132 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:38:30,133 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T03:38:30,134 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T03:38:30,134 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T03:38:30,530 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T03:38:30,531 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:38:30,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:38:30,947 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:38:30,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:38:30,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:38:30,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:30,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:38:30,955 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:38:30,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:38:30,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:30,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:38:30,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:38:30,961 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:38:30,962 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:30,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:38:30,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:38:30,965 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:38:30,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:38:30,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:38:30,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:38:30,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740 2024-11-25T03:38:30,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740 2024-11-25T03:38:30,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:38:30,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:38:30,974 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:38:30,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:38:30,982 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:38:30,984 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61324806, jitterRate=-0.08618918061256409}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:38:30,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732505910947Initializing all the Stores at 1732505910950 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505910950Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505910950Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505910950Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505910950Cleaning up temporary data from old regions at 1732505910973 (+23 ms)Region opened successfully at 1732505910987 (+14 ms) 2024-11-25T03:38:30,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:38:30,988 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:38:30,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:38:30,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:38:30,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:38:30,990 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:38:30,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732505910988Disabling compacts and flushes for region at 1732505910988Disabling writes for close at 1732505910988Writing region close event to WAL at 1732505910989 (+1 ms)Closed at 1732505910990 (+1 ms) 2024-11-25T03:38:30,995 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:38:30,995 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T03:38:31,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:38:31,015 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:38:31,018 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T03:38:31,169 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:32,170 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:33,170 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:34,171 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:35,172 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:36,173 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:37,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T03:38:37,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T03:38:37,173 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:38,174 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:39,175 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:40,176 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:41,177 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:42,178 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:43,179 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:44,179 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:45,180 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:46,181 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:47,181 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:48,182 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:49,183 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:50,183 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:51,184 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:52,184 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:53,185 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:54,186 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:55,187 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:55,633 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:38:56,187 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:57,188 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:58,189 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:38:59,189 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:00,109 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(2659): reportForDuty to master=38a5d2077268,46447,1732505906635 with port=36415, startcode=1732505907601 2024-11-25T03:39:00,111 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46447 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 38a5d2077268,36415,1732505907601 2024-11-25T03:39:00,115 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46447 {}] master.ServerManager(517): Registering regionserver=38a5d2077268,36415,1732505907601 2024-11-25T03:39:00,125 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:39:00,125 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45233 2024-11-25T03:39:00,125 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:39:00,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:39:00,133 DEBUG [RS:0;38a5d2077268:36415 {}] zookeeper.ZKUtil(111): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/38a5d2077268,36415,1732505907601 2024-11-25T03:39:00,133 WARN [RS:0;38a5d2077268:36415 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:39:00,133 INFO [RS:0;38a5d2077268:36415 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:39:00,134 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/WALs/38a5d2077268,36415,1732505907601 2024-11-25T03:39:00,135 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [38a5d2077268,36415,1732505907601] 2024-11-25T03:39:00,183 INFO [RS:0;38a5d2077268:36415 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:39:00,190 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:00,204 INFO [RS:0;38a5d2077268:36415 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:39:00,215 INFO [RS:0;38a5d2077268:36415 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:39:00,215 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,225 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:39:00,238 INFO [RS:0;38a5d2077268:36415 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:39:00,240 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,241 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,244 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,244 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,244 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,245 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,245 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/38a5d2077268:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:39:00,245 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,245 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,245 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,246 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,246 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,246 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/38a5d2077268:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:39:00,246 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/38a5d2077268:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:39:00,246 DEBUG [RS:0;38a5d2077268:36415 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/38a5d2077268:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:39:00,250 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,251 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,251 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,251 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,251 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,251 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,36415,1732505907601-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:39:00,283 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:39:00,286 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,36415,1732505907601-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,286 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,286 INFO [RS:0;38a5d2077268:36415 {}] regionserver.Replication(171): 38a5d2077268,36415,1732505907601 started 2024-11-25T03:39:00,312 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:00,313 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1482): Serving as 38a5d2077268,36415,1732505907601, RpcServer on 38a5d2077268/172.17.0.2:36415, sessionid=0x1014d3414b00001 2024-11-25T03:39:00,314 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:39:00,314 DEBUG [RS:0;38a5d2077268:36415 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 38a5d2077268,36415,1732505907601 2024-11-25T03:39:00,314 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '38a5d2077268,36415,1732505907601' 2024-11-25T03:39:00,314 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:39:00,316 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:39:00,319 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:39:00,319 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:39:00,319 DEBUG [RS:0;38a5d2077268:36415 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 38a5d2077268,36415,1732505907601 2024-11-25T03:39:00,319 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '38a5d2077268,36415,1732505907601' 2024-11-25T03:39:00,319 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:39:00,321 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:39:00,322 DEBUG [RS:0;38a5d2077268:36415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:39:00,322 INFO [RS:0;38a5d2077268:36415 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:39:00,322 INFO [RS:0;38a5d2077268:36415 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:39:01,191 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:02,192 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:03,192 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:04,193 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:05,194 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:06,194 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:07,195 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:08,196 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:09,196 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:10,197 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:11,198 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:12,198 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:13,199 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:14,200 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:15,200 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:16,201 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:17,202 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:18,202 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:19,203 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:20,204 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:21,204 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:22,205 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:23,205 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:24,206 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:25,207 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:25,633 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:39:26,207 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:27,208 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:28,209 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:29,209 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:30,210 WARN [38a5d2077268:46447 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T03:39:30,328 INFO [RS:0;38a5d2077268:36415 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:39:30,334 INFO [RS:0;38a5d2077268:36415 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C36415%2C1732505907601, suffix=, logDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/WALs/38a5d2077268,36415,1732505907601, archiveDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/oldWALs, maxLogs=32 2024-11-25T03:39:30,358 DEBUG [RS:0;38a5d2077268:36415 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/WALs/38a5d2077268,36415,1732505907601/38a5d2077268%2C36415%2C1732505907601.1732505970337, exclude list is [], retry=0 2024-11-25T03:39:30,365 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42577,DS-b1fb3875-2822-4fc5-9221-02816a87253b,DISK] 2024-11-25T03:39:30,370 INFO [RS:0;38a5d2077268:36415 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/WALs/38a5d2077268,36415,1732505907601/38a5d2077268%2C36415%2C1732505907601.1732505970337 2024-11-25T03:39:30,371 DEBUG [RS:0;38a5d2077268:36415 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34425:34425)] 2024-11-25T03:39:30,463 DEBUG [38a5d2077268:46447 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T03:39:30,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:30,487 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 38a5d2077268,36415,1732505907601, state=OPENING 2024-11-25T03:39:30,495 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T03:39:30,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:39:30,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:39:30,503 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:39:30,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:39:30,505 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:39:30,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:39:30,689 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:39:30,693 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50621, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:39:30,709 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:39:30,709 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:39:30,710 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:39:30,714 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=38a5d2077268%2C36415%2C1732505907601.meta, suffix=.meta, logDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/WALs/38a5d2077268,36415,1732505907601, archiveDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/oldWALs, maxLogs=32 2024-11-25T03:39:30,731 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/WALs/38a5d2077268,36415,1732505907601/38a5d2077268%2C36415%2C1732505907601.meta.1732505970716.meta, exclude list is [], retry=0 2024-11-25T03:39:30,736 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42577,DS-b1fb3875-2822-4fc5-9221-02816a87253b,DISK] 2024-11-25T03:39:30,740 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/WALs/38a5d2077268,36415,1732505907601/38a5d2077268%2C36415%2C1732505907601.meta.1732505970716.meta 2024-11-25T03:39:30,740 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34425:34425)] 2024-11-25T03:39:30,741 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:30,742 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:39:30,746 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:39:30,751 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:39:30,756 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:39:30,756 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:30,756 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:39:30,756 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:39:30,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:39:30,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:39:30,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:30,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:39:30,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:39:30,765 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:39:30,765 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:30,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:39:30,766 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:39:30,768 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:39:30,769 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:30,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:39:30,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:39:30,771 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:39:30,771 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:30,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:39:30,772 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:39:30,774 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740 2024-11-25T03:39:30,776 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740 2024-11-25T03:39:30,779 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:39:30,779 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:39:30,781 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:39:30,784 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:39:30,787 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68947087, jitterRate=0.02739165723323822}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:39:30,787 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:39:30,789 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732505970757Writing region info on filesystem at 1732505970757Initializing all the Stores at 1732505970759 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505970759Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505970760 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505970760Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732505970760Cleaning up temporary data from old regions at 1732505970779 (+19 ms)Running coprocessor post-open hooks at 1732505970787 (+8 ms)Region opened successfully at 1732505970789 (+2 ms) 2024-11-25T03:39:30,799 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732505970679 2024-11-25T03:39:30,811 DEBUG [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:39:30,812 INFO [RS_OPEN_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:39:30,814 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:30,817 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 38a5d2077268,36415,1732505907601, state=OPEN 2024-11-25T03:39:30,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:39:30,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:39:30,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:39:30,824 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:39:30,826 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=38a5d2077268,36415,1732505907601 2024-11-25T03:39:30,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T03:39:30,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=38a5d2077268,36415,1732505907601 in 319 msec 2024-11-25T03:39:30,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T03:39:30,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 59.8320 sec 2024-11-25T03:39:30,844 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:39:30,844 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T03:39:30,870 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:30,871 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,36415,1732505907601, seqNum=-1] 2024-11-25T03:39:30,897 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:30,900 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36127, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:30,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 0.978 sec 2024-11-25T03:39:30,930 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732505970930, completionTime=-1 2024-11-25T03:39:30,933 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-25T03:39:30,934 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:39:30,974 INFO [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-25T03:39:30,974 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506030974 2024-11-25T03:39:30,974 INFO [master/38a5d2077268:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506090974 2024-11-25T03:39:30,974 INFO [master/38a5d2077268:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 40 msec 2024-11-25T03:39:30,977 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,46447,1732505906635-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:30,977 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,46447,1732505906635-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:30,977 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,46447,1732505906635-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:30,979 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-38a5d2077268:46447, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:30,979 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:30,980 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:30,987 DEBUG [master/38a5d2077268:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T03:39:31,014 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 63.200sec 2024-11-25T03:39:31,016 INFO [master/38a5d2077268:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T03:39:31,017 INFO [master/38a5d2077268:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T03:39:31,019 INFO [master/38a5d2077268:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T03:39:31,019 INFO [master/38a5d2077268:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T03:39:31,020 INFO [master/38a5d2077268:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T03:39:31,021 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,46447,1732505906635-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:39:31,021 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,46447,1732505906635-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T03:39:31,031 DEBUG [master/38a5d2077268:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T03:39:31,032 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T03:39:31,032 INFO [master/38a5d2077268:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=38a5d2077268,46447,1732505906635-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:39:31,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2560d350, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:31,107 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T03:39:31,107 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T03:39:31,112 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,46447,-1 for getting cluster id 2024-11-25T03:39:31,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:31,125 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68fa995c-d5c8-49e7-9647-a8edabc4bcd7' 2024-11-25T03:39:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:31,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68fa995c-d5c8-49e7-9647-a8edabc4bcd7" 2024-11-25T03:39:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287d24fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:31,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,46447,-1] 2024-11-25T03:39:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:31,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:31,138 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:31,141 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456a05a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:31,142 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:31,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,36415,1732505907601, seqNum=-1] 2024-11-25T03:39:31,150 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:31,153 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:31,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=38a5d2077268,46447,1732505906635 2024-11-25T03:39:31,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a73b085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:31,181 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 38a5d2077268,46447,-1 for getting cluster id 2024-11-25T03:39:31,181 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:39:31,191 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '68fa995c-d5c8-49e7-9647-a8edabc4bcd7' 2024-11-25T03:39:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:39:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "68fa995c-d5c8-49e7-9647-a8edabc4bcd7" 2024-11-25T03:39:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51eae32c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [38a5d2077268,46447,-1] 2024-11-25T03:39:31,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:39:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:39:31,199 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:39:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33176949, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:39:31,230 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=851, ProcessCount=11, AvailableMemoryMB=6595 2024-11-25T03:39:31,239 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:39:31,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 38a5d2077268,46447,1732505906635 2024-11-25T03:39:31,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18689943 2024-11-25T03:39:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:39:31,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:39:31,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:31,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-25T03:39:31,278 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:39:31,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-25T03:39:31,281 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:31,284 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:39:31,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:39:31,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741835_1011 (size=428) 2024-11-25T03:39:31,319 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 091a1e050b8ecf2cad4c31bd0ba3a05c, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:39:31,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741836_1012 (size=79) 2024-11-25T03:39:31,342 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:31,343 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 091a1e050b8ecf2cad4c31bd0ba3a05c, disabling compactions & flushes 2024-11-25T03:39:31,343 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:39:31,343 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:39:31,343 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. after waiting 0 ms 2024-11-25T03:39:31,343 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:39:31,343 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:39:31,343 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 091a1e050b8ecf2cad4c31bd0ba3a05c: Waiting for close lock at 1732505971343Disabling compacts and flushes for region at 1732505971343Disabling writes for close at 1732505971343Writing region close event to WAL at 1732505971343Closed at 1732505971343 2024-11-25T03:39:31,384 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:39:31,390 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732505971384"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505971384"}]},"ts":"1732505971384"} 2024-11-25T03:39:31,397 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:39:31,400 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:39:31,404 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505971400"}]},"ts":"1732505971400"} 2024-11-25T03:39:31,411 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-25T03:39:31,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=091a1e050b8ecf2cad4c31bd0ba3a05c, ASSIGN}] 2024-11-25T03:39:31,426 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=091a1e050b8ecf2cad4c31bd0ba3a05c, ASSIGN 2024-11-25T03:39:31,431 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=091a1e050b8ecf2cad4c31bd0ba3a05c, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:39:31,583 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=091a1e050b8ecf2cad4c31bd0ba3a05c, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:31,588 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=091a1e050b8ecf2cad4c31bd0ba3a05c, ASSIGN because future has completed 2024-11-25T03:39:31,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 091a1e050b8ecf2cad4c31bd0ba3a05c, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:39:31,753 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:39:31,754 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 091a1e050b8ecf2cad4c31bd0ba3a05c, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:31,754 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,754 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:31,755 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,755 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,758 INFO [StoreOpener-091a1e050b8ecf2cad4c31bd0ba3a05c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,760 INFO [StoreOpener-091a1e050b8ecf2cad4c31bd0ba3a05c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 091a1e050b8ecf2cad4c31bd0ba3a05c columnFamilyName testFamily 2024-11-25T03:39:31,760 DEBUG [StoreOpener-091a1e050b8ecf2cad4c31bd0ba3a05c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:31,761 INFO [StoreOpener-091a1e050b8ecf2cad4c31bd0ba3a05c-1 {}] regionserver.HStore(327): Store=091a1e050b8ecf2cad4c31bd0ba3a05c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:31,762 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,763 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,764 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,765 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,765 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,769 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,780 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:39:31,780 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 091a1e050b8ecf2cad4c31bd0ba3a05c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59136863, jitterRate=-0.11879207193851471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:39:31,781 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:39:31,782 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 091a1e050b8ecf2cad4c31bd0ba3a05c: Running coprocessor pre-open hook at 1732505971755Writing region info on filesystem at 1732505971755Initializing all the Stores at 1732505971757 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505971757Cleaning up temporary data from old regions at 1732505971765 (+8 ms)Running coprocessor post-open hooks at 1732505971781 (+16 ms)Region opened successfully at 1732505971782 (+1 ms) 2024-11-25T03:39:31,785 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c., pid=6, masterSystemTime=1732505971744 2024-11-25T03:39:31,789 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:39:31,789 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:39:31,791 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=091a1e050b8ecf2cad4c31bd0ba3a05c, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:31,796 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 091a1e050b8ecf2cad4c31bd0ba3a05c, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:39:31,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T03:39:31,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 091a1e050b8ecf2cad4c31bd0ba3a05c, server=38a5d2077268,36415,1732505907601 in 209 msec 2024-11-25T03:39:31,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T03:39:31,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=091a1e050b8ecf2cad4c31bd0ba3a05c, ASSIGN in 388 msec 2024-11-25T03:39:31,812 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:39:31,812 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505971812"}]},"ts":"1732505971812"} 2024-11-25T03:39:31,817 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-25T03:39:31,819 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:39:31,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 552 msec 2024-11-25T03:39:32,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:39:32,308 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-25T03:39:32,308 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-25T03:39:32,310 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:32,317 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-25T03:39:32,318 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:32,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-25T03:39:32,322 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:39:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=38a5d2077268,36415,1732505907601, seqNum=-1] 2024-11-25T03:39:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:39:32,327 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:39:32,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:39:32,353 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:39:32,359 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-25T03:39:32,364 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-25T03:39:32,369 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-25T03:39:32,371 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-25T03:39:32,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=5052773086642831368 2024-11-25T03:39:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:56294 deadline: 1732505975371, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:39:32,381 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-25T03:39:32,383 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-25T03:39:32,384 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:32,387 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:32,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=5052773086642831370 2024-11-25T03:39:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:56294 deadline: 1732505975386, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:39:32,392 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:32,393 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=5052773086642831371 2024-11-25T03:39:32,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:56294 deadline: 1732505975392, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:39:32,397 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:32,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=5052773086642831372 2024-11-25T03:39:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:56294 deadline: 1732505975396, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:39:32,403 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:32,420 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-693575612_22 at /127.0.0.1:37550 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=851 (was 851), ProcessCount=11 (was 11), AvailableMemoryMB=6546 (was 6595) 2024-11-25T03:39:32,430 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=851, ProcessCount=11, AvailableMemoryMB=6546 2024-11-25T03:39:32,436 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:32,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-25T03:39:32,441 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:39:32,442 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:32,442 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-25T03:39:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:39:32,444 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:39:32,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741837_1013 (size=414) 2024-11-25T03:39:32,461 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3d1f0936f96645ecc5d8c057f7aae55e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:39:32,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741838_1014 (size=65) 2024-11-25T03:39:32,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:32,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 3d1f0936f96645ecc5d8c057f7aae55e, disabling compactions & flushes 2024-11-25T03:39:32,490 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:39:32,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:39:32,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. after waiting 0 ms 2024-11-25T03:39:32,490 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:39:32,490 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:39:32,491 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3d1f0936f96645ecc5d8c057f7aae55e: Waiting for close lock at 1732505972490Disabling compacts and flushes for region at 1732505972490Disabling writes for close at 1732505972490Writing region close event to WAL at 1732505972490Closed at 1732505972490 2024-11-25T03:39:32,493 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:39:32,493 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732505972493"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505972493"}]},"ts":"1732505972493"} 2024-11-25T03:39:32,497 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:39:32,499 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:39:32,499 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505972499"}]},"ts":"1732505972499"} 2024-11-25T03:39:32,503 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-25T03:39:32,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3d1f0936f96645ecc5d8c057f7aae55e, ASSIGN}] 2024-11-25T03:39:32,506 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3d1f0936f96645ecc5d8c057f7aae55e, ASSIGN 2024-11-25T03:39:32,509 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3d1f0936f96645ecc5d8c057f7aae55e, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:39:32,660 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=3d1f0936f96645ecc5d8c057f7aae55e, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:32,665 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3d1f0936f96645ecc5d8c057f7aae55e, ASSIGN because future has completed 2024-11-25T03:39:32,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d1f0936f96645ecc5d8c057f7aae55e, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:39:32,828 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:39:32,828 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d1f0936f96645ecc5d8c057f7aae55e, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:32,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:32,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,829 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,832 INFO [StoreOpener-3d1f0936f96645ecc5d8c057f7aae55e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,834 INFO [StoreOpener-3d1f0936f96645ecc5d8c057f7aae55e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d1f0936f96645ecc5d8c057f7aae55e columnFamilyName testFamily 2024-11-25T03:39:32,835 DEBUG [StoreOpener-3d1f0936f96645ecc5d8c057f7aae55e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:32,835 INFO [StoreOpener-3d1f0936f96645ecc5d8c057f7aae55e-1 {}] regionserver.HStore(327): Store=3d1f0936f96645ecc5d8c057f7aae55e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:32,836 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,838 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,843 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:39:32,849 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 3d1f0936f96645ecc5d8c057f7aae55e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68422148, jitterRate=0.019569456577301025}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:39:32,849 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:39:32,851 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3d1f0936f96645ecc5d8c057f7aae55e: Running coprocessor pre-open hook at 1732505972830Writing region info on filesystem at 1732505972830Initializing all the Stores at 1732505972831 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505972831Cleaning up temporary data from old regions at 1732505972840 (+9 ms)Running coprocessor post-open hooks at 1732505972849 (+9 ms)Region opened successfully at 1732505972851 (+2 ms) 2024-11-25T03:39:32,853 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., pid=9, masterSystemTime=1732505972820 2024-11-25T03:39:32,857 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:39:32,857 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:39:32,858 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=3d1f0936f96645ecc5d8c057f7aae55e, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:32,862 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3d1f0936f96645ecc5d8c057f7aae55e, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:39:32,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-25T03:39:32,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 3d1f0936f96645ecc5d8c057f7aae55e, server=38a5d2077268,36415,1732505907601 in 198 msec 2024-11-25T03:39:32,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T03:39:32,875 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3d1f0936f96645ecc5d8c057f7aae55e, ASSIGN in 365 msec 2024-11-25T03:39:32,878 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:39:32,878 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505972878"}]},"ts":"1732505972878"} 2024-11-25T03:39:32,881 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-25T03:39:32,883 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:39:32,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 447 msec 2024-11-25T03:39:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-25T03:39:33,457 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-25T03:39:33,457 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-25T03:39:33,457 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:33,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-25T03:39:33,462 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:33,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-25T03:39:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:39:33,480 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:39:33,480 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-25T03:39:33,483 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T03:39:33,485 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-25T03:39:36,304 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:39:36,357 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-25T03:39:36,361 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T03:39:36,363 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-25T03:39:37,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:39:37,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T03:39:37,155 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:39:37,155 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T03:39:37,155 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-25T03:39:37,155 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:39:37,156 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-25T03:39:37,156 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-25T03:39:39,497 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., hostname=38a5d2077268,36415,1732505907601, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., hostname=38a5d2077268,36415,1732505907601, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=38a5d2077268:36415 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-25T03:39:39,498 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., hostname=38a5d2077268,36415,1732505907601, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-25T03:39:39,498 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., hostname=38a5d2077268,36415,1732505907601, seqNum=2 from cache 2024-11-25T03:39:39,510 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=812 (was 851), ProcessCount=11 (was 11), AvailableMemoryMB=6521 (was 6546) 2024-11-25T03:39:39,519 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=812, ProcessCount=11, AvailableMemoryMB=6521 2024-11-25T03:39:39,522 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:39,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-25T03:39:39,527 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:39:39,527 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:39,527 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-25T03:39:39,529 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:39:39,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-25T03:39:39,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741839_1015 (size=412) 2024-11-25T03:39:39,945 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1811ab78326ea2bfc621a27fb94cc3f7, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:39:39,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741840_1016 (size=63) 2024-11-25T03:39:39,959 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:39,959 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 1811ab78326ea2bfc621a27fb94cc3f7, disabling compactions & flushes 2024-11-25T03:39:39,959 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:39:39,959 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:39:39,959 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. after waiting 0 ms 2024-11-25T03:39:39,959 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:39:39,960 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:39:39,960 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1811ab78326ea2bfc621a27fb94cc3f7: Waiting for close lock at 1732505979959Disabling compacts and flushes for region at 1732505979959Disabling writes for close at 1732505979959Writing region close event to WAL at 1732505979959Closed at 1732505979959 2024-11-25T03:39:39,962 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:39:39,962 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732505979962"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505979962"}]},"ts":"1732505979962"} 2024-11-25T03:39:39,966 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:39:39,967 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:39:39,968 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505979968"}]},"ts":"1732505979968"} 2024-11-25T03:39:39,971 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-25T03:39:39,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1811ab78326ea2bfc621a27fb94cc3f7, ASSIGN}] 2024-11-25T03:39:39,973 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1811ab78326ea2bfc621a27fb94cc3f7, ASSIGN 2024-11-25T03:39:39,975 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1811ab78326ea2bfc621a27fb94cc3f7, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:39:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-25T03:39:40,125 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1811ab78326ea2bfc621a27fb94cc3f7, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:40,130 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1811ab78326ea2bfc621a27fb94cc3f7, ASSIGN because future has completed 2024-11-25T03:39:40,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1811ab78326ea2bfc621a27fb94cc3f7, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:39:40,292 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:39:40,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 1811ab78326ea2bfc621a27fb94cc3f7, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:40,293 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,293 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:40,293 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,293 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,296 INFO [StoreOpener-1811ab78326ea2bfc621a27fb94cc3f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,298 INFO [StoreOpener-1811ab78326ea2bfc621a27fb94cc3f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1811ab78326ea2bfc621a27fb94cc3f7 columnFamilyName testFamily 2024-11-25T03:39:40,298 DEBUG [StoreOpener-1811ab78326ea2bfc621a27fb94cc3f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:40,299 INFO [StoreOpener-1811ab78326ea2bfc621a27fb94cc3f7-1 {}] regionserver.HStore(327): Store=1811ab78326ea2bfc621a27fb94cc3f7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:40,300 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,302 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,306 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,310 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:39:40,311 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 1811ab78326ea2bfc621a27fb94cc3f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66227787, jitterRate=-0.013129070401191711}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:39:40,311 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:39:40,312 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 1811ab78326ea2bfc621a27fb94cc3f7: Running coprocessor pre-open hook at 1732505980293Writing region info on filesystem at 1732505980293Initializing all the Stores at 1732505980295 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505980295Cleaning up temporary data from old regions at 1732505980302 (+7 ms)Running coprocessor post-open hooks at 1732505980311 (+9 ms)Region opened successfully at 1732505980312 (+1 ms) 2024-11-25T03:39:40,314 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7., pid=12, masterSystemTime=1732505980285 2024-11-25T03:39:40,318 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:39:40,319 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:39:40,319 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1811ab78326ea2bfc621a27fb94cc3f7, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:40,324 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1811ab78326ea2bfc621a27fb94cc3f7, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:39:40,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-25T03:39:40,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 1811ab78326ea2bfc621a27fb94cc3f7, server=38a5d2077268,36415,1732505907601 in 195 msec 2024-11-25T03:39:40,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T03:39:40,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=1811ab78326ea2bfc621a27fb94cc3f7, ASSIGN in 360 msec 2024-11-25T03:39:40,338 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:39:40,338 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505980338"}]},"ts":"1732505980338"} 2024-11-25T03:39:40,341 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-25T03:39:40,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:39:40,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 821 msec 2024-11-25T03:39:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-25T03:39:40,536 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-25T03:39:40,536 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-25T03:39:40,537 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:40,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-25T03:39:40,542 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:40,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-25T03:39:40,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:39:40,562 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:39:40,563 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-25T03:39:40,567 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T03:39:40,570 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T03:39:40,571 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T03:39:40,572 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-25T03:39:42,364 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-25T03:39:45,795 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:39:46,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36415 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831381' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:39:46,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36415 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:56294 deadline: 1732505992581, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831381' does not exist 2024-11-25T03:39:46,585 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 38a5d2077268,36415,1732505907601 for closing scanner id = 5052773086642831381 for 1811ab78326ea2bfc621a27fb94cc3f7 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831381' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:46,588 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=771 (was 812), ProcessCount=11 (was 11), AvailableMemoryMB=6522 (was 6521) - AvailableMemoryMB LEAK? - 2024-11-25T03:39:46,596 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=771, ProcessCount=11, AvailableMemoryMB=6521 2024-11-25T03:39:46,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-25T03:39:46,602 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:39:46,602 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:46,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-25T03:39:46,603 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:39:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:39:46,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741841_1017 (size=423) 2024-11-25T03:39:47,014 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5cc028e6e24e6f88d9a2913e959758ad, NAME => 'testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:39:47,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741842_1018 (size=74) 2024-11-25T03:39:47,073 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-25T03:39:47,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-25T03:39:47,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:39:47,423 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:47,423 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 5cc028e6e24e6f88d9a2913e959758ad, disabling compactions & flushes 2024-11-25T03:39:47,423 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:39:47,423 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:39:47,424 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. after waiting 0 ms 2024-11-25T03:39:47,424 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:39:47,424 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:39:47,424 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5cc028e6e24e6f88d9a2913e959758ad: Waiting for close lock at 1732505987423Disabling compacts and flushes for region at 1732505987423Disabling writes for close at 1732505987424 (+1 ms)Writing region close event to WAL at 1732505987424Closed at 1732505987424 2024-11-25T03:39:47,426 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:39:47,426 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732505987426"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505987426"}]},"ts":"1732505987426"} 2024-11-25T03:39:47,429 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:39:47,430 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:39:47,431 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505987430"}]},"ts":"1732505987430"} 2024-11-25T03:39:47,433 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-25T03:39:47,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5cc028e6e24e6f88d9a2913e959758ad, ASSIGN}] 2024-11-25T03:39:47,434 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5cc028e6e24e6f88d9a2913e959758ad, ASSIGN 2024-11-25T03:39:47,436 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5cc028e6e24e6f88d9a2913e959758ad, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:39:47,587 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5cc028e6e24e6f88d9a2913e959758ad, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:47,590 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5cc028e6e24e6f88d9a2913e959758ad, ASSIGN because future has completed 2024-11-25T03:39:47,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cc028e6e24e6f88d9a2913e959758ad, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:39:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:39:47,749 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:39:47,749 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cc028e6e24e6f88d9a2913e959758ad, NAME => 'testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:47,749 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,749 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:47,750 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,750 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,752 INFO [StoreOpener-5cc028e6e24e6f88d9a2913e959758ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,754 INFO [StoreOpener-5cc028e6e24e6f88d9a2913e959758ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cc028e6e24e6f88d9a2913e959758ad columnFamilyName testFamily 2024-11-25T03:39:47,755 DEBUG [StoreOpener-5cc028e6e24e6f88d9a2913e959758ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:47,755 INFO [StoreOpener-5cc028e6e24e6f88d9a2913e959758ad-1 {}] regionserver.HStore(327): Store=5cc028e6e24e6f88d9a2913e959758ad/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:47,756 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,757 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,757 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,758 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,758 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,760 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,763 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:39:47,764 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 5cc028e6e24e6f88d9a2913e959758ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65857467, jitterRate=-0.018647268414497375}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:39:47,764 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:39:47,764 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 5cc028e6e24e6f88d9a2913e959758ad: Running coprocessor pre-open hook at 1732505987750Writing region info on filesystem at 1732505987750Initializing all the Stores at 1732505987751 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505987751Cleaning up temporary data from old regions at 1732505987758 (+7 ms)Running coprocessor post-open hooks at 1732505987764 (+6 ms)Region opened successfully at 1732505987764 2024-11-25T03:39:47,766 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad., pid=15, masterSystemTime=1732505987743 2024-11-25T03:39:47,769 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:39:47,769 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:39:47,770 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5cc028e6e24e6f88d9a2913e959758ad, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:47,773 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cc028e6e24e6f88d9a2913e959758ad, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:39:47,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-25T03:39:47,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5cc028e6e24e6f88d9a2913e959758ad, server=38a5d2077268,36415,1732505907601 in 184 msec 2024-11-25T03:39:47,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T03:39:47,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=5cc028e6e24e6f88d9a2913e959758ad, ASSIGN in 345 msec 2024-11-25T03:39:47,783 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:39:47,783 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505987783"}]},"ts":"1732505987783"} 2024-11-25T03:39:47,786 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-25T03:39:47,787 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:39:47,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.1880 sec 2024-11-25T03:39:48,366 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-25T03:39:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-25T03:39:49,637 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-25T03:39:49,637 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-25T03:39:49,637 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:49,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-25T03:39:49,642 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:49,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-25T03:39:49,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:39:49,657 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:39:49,657 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-25T03:39:49,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-25T03:39:49,663 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-25T03:39:49,665 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-25T03:39:49,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=5052773086642831385 2024-11-25T03:39:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:56294 deadline: 1732505992664, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:39:49,668 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-25T03:39:49,670 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-25T03:39:49,670 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:49,672 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:49,672 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=5052773086642831387 2024-11-25T03:39:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:56294 deadline: 1732505992671, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:39:49,676 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:49,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=5052773086642831388 2024-11-25T03:39:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:56294 deadline: 1732505992675, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:39:49,680 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:49,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=5052773086642831389 2024-11-25T03:39:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:56294 deadline: 1732505992679, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-25T03:39:49,684 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-25T03:39:49,694 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-693575612_22 at /127.0.0.1:58966 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=709 (was 771), ProcessCount=11 (was 11), AvailableMemoryMB=6516 (was 6521) 2024-11-25T03:39:49,704 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=709, ProcessCount=11, AvailableMemoryMB=6516 2024-11-25T03:39:49,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-25T03:39:49,709 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:39:49,709 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:49,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-25T03:39:49,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:39:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T03:39:49,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741843_1019 (size=418) 2024-11-25T03:39:50,122 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 136205dd4c3406f1b0ed50672de492f7, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:39:50,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741844_1020 (size=69) 2024-11-25T03:39:50,531 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:50,531 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 136205dd4c3406f1b0ed50672de492f7, disabling compactions & flushes 2024-11-25T03:39:50,531 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:39:50,531 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:39:50,531 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. after waiting 0 ms 2024-11-25T03:39:50,531 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:39:50,531 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:39:50,531 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 136205dd4c3406f1b0ed50672de492f7: Waiting for close lock at 1732505990531Disabling compacts and flushes for region at 1732505990531Disabling writes for close at 1732505990531Writing region close event to WAL at 1732505990531Closed at 1732505990531 2024-11-25T03:39:50,533 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:39:50,533 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732505990533"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505990533"}]},"ts":"1732505990533"} 2024-11-25T03:39:50,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:39:50,538 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:39:50,538 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505990538"}]},"ts":"1732505990538"} 2024-11-25T03:39:50,541 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-25T03:39:50,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=136205dd4c3406f1b0ed50672de492f7, ASSIGN}] 2024-11-25T03:39:50,543 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=136205dd4c3406f1b0ed50672de492f7, ASSIGN 2024-11-25T03:39:50,544 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=136205dd4c3406f1b0ed50672de492f7, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:39:50,695 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=136205dd4c3406f1b0ed50672de492f7, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:50,698 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=136205dd4c3406f1b0ed50672de492f7, ASSIGN because future has completed 2024-11-25T03:39:50,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 136205dd4c3406f1b0ed50672de492f7, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:39:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T03:39:50,858 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:39:50,858 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 136205dd4c3406f1b0ed50672de492f7, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:50,858 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,858 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:50,859 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,859 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,861 INFO [StoreOpener-136205dd4c3406f1b0ed50672de492f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,863 INFO [StoreOpener-136205dd4c3406f1b0ed50672de492f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 136205dd4c3406f1b0ed50672de492f7 columnFamilyName testFamily 2024-11-25T03:39:50,863 DEBUG [StoreOpener-136205dd4c3406f1b0ed50672de492f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:50,863 INFO [StoreOpener-136205dd4c3406f1b0ed50672de492f7-1 {}] regionserver.HStore(327): Store=136205dd4c3406f1b0ed50672de492f7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:50,864 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,865 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,865 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,866 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,866 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,868 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,871 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:39:50,872 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 136205dd4c3406f1b0ed50672de492f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70537289, jitterRate=0.05108751356601715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:39:50,872 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:39:50,873 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 136205dd4c3406f1b0ed50672de492f7: Running coprocessor pre-open hook at 1732505990859Writing region info on filesystem at 1732505990859Initializing all the Stores at 1732505990860 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505990860Cleaning up temporary data from old regions at 1732505990866 (+6 ms)Running coprocessor post-open hooks at 1732505990872 (+6 ms)Region opened successfully at 1732505990873 (+1 ms) 2024-11-25T03:39:50,874 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., pid=18, masterSystemTime=1732505990852 2024-11-25T03:39:50,877 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:39:50,877 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:39:50,878 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=136205dd4c3406f1b0ed50672de492f7, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:50,881 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 136205dd4c3406f1b0ed50672de492f7, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:39:50,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-25T03:39:50,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 136205dd4c3406f1b0ed50672de492f7, server=38a5d2077268,36415,1732505907601 in 184 msec 2024-11-25T03:39:50,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-25T03:39:50,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=136205dd4c3406f1b0ed50672de492f7, ASSIGN in 345 msec 2024-11-25T03:39:50,890 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:39:50,891 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505990890"}]},"ts":"1732505990890"} 2024-11-25T03:39:50,893 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-25T03:39:50,894 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:39:50,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1880 sec 2024-11-25T03:39:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-25T03:39:52,747 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-25T03:39:52,747 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-25T03:39:52,747 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:52,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-25T03:39:52,751 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:52,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-25T03:39:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:39:52,765 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:39:52,765 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-25T03:39:52,767 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T03:39:52,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T03:39:53,251 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:39:53,769 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=38a5d2077268:36415 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-11-25T03:39:53,769 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1001ms, rpcTimeout=1000ms 2024-11-25T03:39:53,769 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2 from cache 2024-11-25T03:39:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-25T03:39:54,367 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-25T03:39:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:39:54,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T03:39:55,634 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:39:55,769 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=38a5d2077268:36415 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=979ms 2024-11-25T03:39:55,770 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=979ms 2024-11-25T03:39:55,770 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., hostname=38a5d2077268,36415,1732505907601, seqNum=2 from cache 2024-11-25T03:39:55,782 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=668 (was 709), ProcessCount=11 (was 11), AvailableMemoryMB=7815 (was 6516) - AvailableMemoryMB LEAK? - 2024-11-25T03:39:55,793 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=668, ProcessCount=11, AvailableMemoryMB=7814 2024-11-25T03:39:55,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-25T03:39:55,801 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:39:55,801 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:55,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-25T03:39:55,803 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:39:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:39:55,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741845_1021 (size=416) 2024-11-25T03:39:56,214 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0d484ec531b38a11fd8d9844c8403ad3, NAME => 'testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:39:56,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741846_1022 (size=67) 2024-11-25T03:39:56,224 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:56,224 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0d484ec531b38a11fd8d9844c8403ad3, disabling compactions & flushes 2024-11-25T03:39:56,224 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:39:56,225 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:39:56,225 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. after waiting 0 ms 2024-11-25T03:39:56,225 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:39:56,225 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:39:56,225 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0d484ec531b38a11fd8d9844c8403ad3: Waiting for close lock at 1732505996224Disabling compacts and flushes for region at 1732505996224Disabling writes for close at 1732505996225 (+1 ms)Writing region close event to WAL at 1732505996225Closed at 1732505996225 2024-11-25T03:39:56,227 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:39:56,227 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732505996227"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732505996227"}]},"ts":"1732505996227"} 2024-11-25T03:39:56,231 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:39:56,232 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:39:56,233 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505996232"}]},"ts":"1732505996232"} 2024-11-25T03:39:56,235 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-25T03:39:56,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0d484ec531b38a11fd8d9844c8403ad3, ASSIGN}] 2024-11-25T03:39:56,238 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0d484ec531b38a11fd8d9844c8403ad3, ASSIGN 2024-11-25T03:39:56,239 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0d484ec531b38a11fd8d9844c8403ad3, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:39:56,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-25T03:39:56,390 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=0d484ec531b38a11fd8d9844c8403ad3, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:56,393 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0d484ec531b38a11fd8d9844c8403ad3, ASSIGN because future has completed 2024-11-25T03:39:56,394 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d484ec531b38a11fd8d9844c8403ad3, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:39:56,554 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:39:56,555 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 0d484ec531b38a11fd8d9844c8403ad3, NAME => 'testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:39:56,556 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,556 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:39:56,556 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,556 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,559 INFO [StoreOpener-0d484ec531b38a11fd8d9844c8403ad3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,561 INFO [StoreOpener-0d484ec531b38a11fd8d9844c8403ad3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d484ec531b38a11fd8d9844c8403ad3 columnFamilyName testFamily 2024-11-25T03:39:56,561 DEBUG [StoreOpener-0d484ec531b38a11fd8d9844c8403ad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:56,562 INFO [StoreOpener-0d484ec531b38a11fd8d9844c8403ad3-1 {}] regionserver.HStore(327): Store=0d484ec531b38a11fd8d9844c8403ad3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:39:56,563 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,564 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,565 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,565 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,566 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,569 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,577 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:39:56,578 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 0d484ec531b38a11fd8d9844c8403ad3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75334913, jitterRate=0.12257768213748932}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:39:56,578 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:39:56,579 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 0d484ec531b38a11fd8d9844c8403ad3: Running coprocessor pre-open hook at 1732505996556Writing region info on filesystem at 1732505996556Initializing all the Stores at 1732505996558 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732505996558Cleaning up temporary data from old regions at 1732505996566 (+8 ms)Running coprocessor post-open hooks at 1732505996578 (+12 ms)Region opened successfully at 1732505996579 (+1 ms) 2024-11-25T03:39:56,580 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3., pid=21, masterSystemTime=1732505996547 2024-11-25T03:39:56,584 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:39:56,584 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:39:56,585 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=0d484ec531b38a11fd8d9844c8403ad3, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:39:56,589 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0d484ec531b38a11fd8d9844c8403ad3, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:39:56,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-25T03:39:56,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 0d484ec531b38a11fd8d9844c8403ad3, server=38a5d2077268,36415,1732505907601 in 196 msec 2024-11-25T03:39:56,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-25T03:39:56,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=0d484ec531b38a11fd8d9844c8403ad3, ASSIGN in 357 msec 2024-11-25T03:39:56,597 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:39:56,598 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732505996597"}]},"ts":"1732505996597"} 2024-11-25T03:39:56,600 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-25T03:39:56,601 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:39:56,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 804 msec 2024-11-25T03:39:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-25T03:39:56,816 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-25T03:39:56,816 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-25T03:39:56,817 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:56,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-25T03:39:56,821 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:39:56,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-25T03:39:56,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:39:56,837 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:39:56,838 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-25T03:39:56,841 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T03:39:56,843 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T03:39:56,845 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T03:39:56,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-25T03:39:57,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-25T03:39:57,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-25T03:39:57,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-25T03:39:57,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-25T03:39:57,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-25T03:39:57,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:39:59,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36415 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831400' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:39:59,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36415 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:56294 deadline: 1732506000852, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831400' does not exist 2024-11-25T03:39:59,853 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 38a5d2077268,36415,1732505907601 for closing scanner id = 5052773086642831400 for 0d484ec531b38a11fd8d9844c8403ad3 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831400' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:39:59,858 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-693575612_22 at /127.0.0.1:53822 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=631 (was 668), ProcessCount=11 (was 11), AvailableMemoryMB=7437 (was 7814) 2024-11-25T03:39:59,867 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=631, ProcessCount=11, AvailableMemoryMB=7436 2024-11-25T03:39:59,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:39:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-25T03:39:59,873 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:39:59,873 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:39:59,874 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-25T03:39:59,875 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:39:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T03:39:59,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741847_1023 (size=411) 2024-11-25T03:40:00,286 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d67a5cdf20dcb39f7da0b7f84a6b00e7, NAME => 'testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:40:00,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741848_1024 (size=62) 2024-11-25T03:40:00,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-25T03:40:00,368 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-25T03:40:00,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:40:00,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing d67a5cdf20dcb39f7da0b7f84a6b00e7, disabling compactions & flushes 2024-11-25T03:40:00,695 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:00,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:00,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. after waiting 0 ms 2024-11-25T03:40:00,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:00,695 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:00,695 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for d67a5cdf20dcb39f7da0b7f84a6b00e7: Waiting for close lock at 1732506000695Disabling compacts and flushes for region at 1732506000695Disabling writes for close at 1732506000695Writing region close event to WAL at 1732506000695Closed at 1732506000695 2024-11-25T03:40:00,697 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:40:00,698 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732506000697"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506000697"}]},"ts":"1732506000697"} 2024-11-25T03:40:00,701 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:40:00,702 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:40:00,703 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506000702"}]},"ts":"1732506000702"} 2024-11-25T03:40:00,705 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-25T03:40:00,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d67a5cdf20dcb39f7da0b7f84a6b00e7, ASSIGN}] 2024-11-25T03:40:00,707 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d67a5cdf20dcb39f7da0b7f84a6b00e7, ASSIGN 2024-11-25T03:40:00,708 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d67a5cdf20dcb39f7da0b7f84a6b00e7, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:40:00,859 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d67a5cdf20dcb39f7da0b7f84a6b00e7, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:40:00,863 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d67a5cdf20dcb39f7da0b7f84a6b00e7, ASSIGN because future has completed 2024-11-25T03:40:00,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d67a5cdf20dcb39f7da0b7f84a6b00e7, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:40:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T03:40:01,021 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:01,022 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => d67a5cdf20dcb39f7da0b7f84a6b00e7, NAME => 'testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:40:01,022 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,022 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:40:01,022 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,022 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,024 INFO [StoreOpener-d67a5cdf20dcb39f7da0b7f84a6b00e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,026 INFO [StoreOpener-d67a5cdf20dcb39f7da0b7f84a6b00e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d67a5cdf20dcb39f7da0b7f84a6b00e7 columnFamilyName testFamily 2024-11-25T03:40:01,026 DEBUG [StoreOpener-d67a5cdf20dcb39f7da0b7f84a6b00e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:40:01,027 INFO [StoreOpener-d67a5cdf20dcb39f7da0b7f84a6b00e7-1 {}] regionserver.HStore(327): Store=d67a5cdf20dcb39f7da0b7f84a6b00e7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:40:01,027 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,028 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,028 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,029 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,029 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,031 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,034 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:40:01,034 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened d67a5cdf20dcb39f7da0b7f84a6b00e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62487966, jitterRate=-0.06885674595832825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:40:01,035 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:01,035 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for d67a5cdf20dcb39f7da0b7f84a6b00e7: Running coprocessor pre-open hook at 1732506001022Writing region info on filesystem at 1732506001022Initializing all the Stores at 1732506001024 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506001024Cleaning up temporary data from old regions at 1732506001029 (+5 ms)Running coprocessor post-open hooks at 1732506001035 (+6 ms)Region opened successfully at 1732506001035 2024-11-25T03:40:01,036 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7., pid=24, masterSystemTime=1732506001016 2024-11-25T03:40:01,039 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:01,039 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:01,039 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=d67a5cdf20dcb39f7da0b7f84a6b00e7, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:40:01,042 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure d67a5cdf20dcb39f7da0b7f84a6b00e7, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:40:01,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-25T03:40:01,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure d67a5cdf20dcb39f7da0b7f84a6b00e7, server=38a5d2077268,36415,1732505907601 in 180 msec 2024-11-25T03:40:01,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-25T03:40:01,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=d67a5cdf20dcb39f7da0b7f84a6b00e7, ASSIGN in 341 msec 2024-11-25T03:40:01,049 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:40:01,050 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506001049"}]},"ts":"1732506001049"} 2024-11-25T03:40:01,052 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-25T03:40:01,053 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:40:01,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1830 sec 2024-11-25T03:40:02,057 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:40:02,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-25T03:40:02,906 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-25T03:40:02,906 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-25T03:40:02,906 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:40:02,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-25T03:40:02,911 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:40:02,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-25T03:40:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:40:02,926 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:40:02,926 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-25T03:40:02,929 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T03:40:02,932 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T03:40:02,933 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T03:40:02,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-25T03:40:05,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36415 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831404' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:40:05,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36415 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:56294 deadline: 1732506006942, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831404' does not exist 2024-11-25T03:40:05,943 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 38a5d2077268,36415,1732505907601 for closing scanner id = 5052773086642831404 for d67a5cdf20dcb39f7da0b7f84a6b00e7 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831404' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:40:05,949 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 223), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=620 (was 631), ProcessCount=11 (was 11), AvailableMemoryMB=7244 (was 7436) 2024-11-25T03:40:05,956 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=620, ProcessCount=11, AvailableMemoryMB=7244 2024-11-25T03:40:05,958 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:40:05,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-25T03:40:05,961 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:40:05,961 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:40:05,962 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-25T03:40:05,963 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:40:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:40:05,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741849_1025 (size=419) 2024-11-25T03:40:06,370 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-25T03:40:06,374 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9f52f55338ad433d07df92f3d74b5266, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:40:06,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741850_1026 (size=70) 2024-11-25T03:40:06,434 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-25T03:40:06,787 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:40:06,787 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 9f52f55338ad433d07df92f3d74b5266, disabling compactions & flushes 2024-11-25T03:40:06,787 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:06,787 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:06,787 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. after waiting 0 ms 2024-11-25T03:40:06,787 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:06,787 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:06,787 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9f52f55338ad433d07df92f3d74b5266: Waiting for close lock at 1732506006787Disabling compacts and flushes for region at 1732506006787Disabling writes for close at 1732506006787Writing region close event to WAL at 1732506006787Closed at 1732506006787 2024-11-25T03:40:06,789 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:40:06,789 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732506006789"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506006789"}]},"ts":"1732506006789"} 2024-11-25T03:40:06,792 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:40:06,793 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:40:06,793 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506006793"}]},"ts":"1732506006793"} 2024-11-25T03:40:06,795 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-25T03:40:06,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9f52f55338ad433d07df92f3d74b5266, ASSIGN}] 2024-11-25T03:40:06,797 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9f52f55338ad433d07df92f3d74b5266, ASSIGN 2024-11-25T03:40:06,798 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9f52f55338ad433d07df92f3d74b5266, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:40:06,949 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=9f52f55338ad433d07df92f3d74b5266, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:40:06,952 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9f52f55338ad433d07df92f3d74b5266, ASSIGN because future has completed 2024-11-25T03:40:06,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f52f55338ad433d07df92f3d74b5266, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:40:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:40:07,110 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:07,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 9f52f55338ad433d07df92f3d74b5266, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:40:07,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:40:07,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,113 INFO [StoreOpener-9f52f55338ad433d07df92f3d74b5266-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,115 INFO [StoreOpener-9f52f55338ad433d07df92f3d74b5266-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9f52f55338ad433d07df92f3d74b5266 columnFamilyName testFamily 2024-11-25T03:40:07,115 DEBUG [StoreOpener-9f52f55338ad433d07df92f3d74b5266-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:40:07,115 INFO [StoreOpener-9f52f55338ad433d07df92f3d74b5266-1 {}] regionserver.HStore(327): Store=9f52f55338ad433d07df92f3d74b5266/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:40:07,116 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,117 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,117 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,118 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,118 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,121 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:40:07,124 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 9f52f55338ad433d07df92f3d74b5266; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74371562, jitterRate=0.10822263360023499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:40:07,124 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:07,125 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 9f52f55338ad433d07df92f3d74b5266: Running coprocessor pre-open hook at 1732506007111Writing region info on filesystem at 1732506007111Initializing all the Stores at 1732506007113 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506007113Cleaning up temporary data from old regions at 1732506007118 (+5 ms)Running coprocessor post-open hooks at 1732506007124 (+6 ms)Region opened successfully at 1732506007124 2024-11-25T03:40:07,126 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., pid=27, masterSystemTime=1732506007105 2024-11-25T03:40:07,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:07,128 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:07,129 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=9f52f55338ad433d07df92f3d74b5266, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:40:07,131 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9f52f55338ad433d07df92f3d74b5266, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:40:07,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-25T03:40:07,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 9f52f55338ad433d07df92f3d74b5266, server=38a5d2077268,36415,1732505907601 in 180 msec 2024-11-25T03:40:07,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-25T03:40:07,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=9f52f55338ad433d07df92f3d74b5266, ASSIGN in 340 msec 2024-11-25T03:40:07,138 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:40:07,138 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506007138"}]},"ts":"1732506007138"} 2024-11-25T03:40:07,140 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-25T03:40:07,141 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:40:07,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1820 sec 2024-11-25T03:40:07,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-25T03:40:07,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-25T03:40:07,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-25T03:40:07,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:40:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-25T03:40:08,987 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-25T03:40:08,987 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-25T03:40:08,987 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:40:08,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-25T03:40:08,991 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:40:08,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-25T03:40:08,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:40:09,003 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:40:09,004 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-25T03:40:09,006 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T03:40:09,007 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-25T03:40:11,661 INFO [master/38a5d2077268:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T03:40:11,661 INFO [master/38a5d2077268:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T03:40:12,632 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:40:12,660 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-25T03:40:15,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., hostname=38a5d2077268,36415,1732505907601, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., hostname=38a5d2077268,36415,1732505907601, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=38a5d2077268:36415 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-25T03:40:15,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., hostname=38a5d2077268,36415,1732505907601, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6003ms, rpcTimeout=5999ms 2024-11-25T03:40:15,009 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., hostname=38a5d2077268,36415,1732505907601, seqNum=2 from cache 2024-11-25T03:40:15,019 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=525 (was 620), ProcessCount=11 (was 11), AvailableMemoryMB=7174 (was 7244) 2024-11-25T03:40:15,029 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=525, ProcessCount=11, AvailableMemoryMB=7172 2024-11-25T03:40:15,032 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:40:15,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-25T03:40:15,035 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:40:15,035 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:40:15,035 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-25T03:40:15,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:40:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T03:40:15,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741851_1027 (size=407) 2024-11-25T03:40:15,446 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 251c13835aa3825db727b602ec851c98, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:40:15,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741852_1028 (size=58) 2024-11-25T03:40:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-25T03:40:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:40:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 251c13835aa3825db727b602ec851c98, disabling compactions & flushes 2024-11-25T03:40:15,856 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. after waiting 0 ms 2024-11-25T03:40:15,856 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:15,857 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:15,857 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 251c13835aa3825db727b602ec851c98: Waiting for close lock at 1732506015856Disabling compacts and flushes for region at 1732506015856Disabling writes for close at 1732506015856Writing region close event to WAL at 1732506015856Closed at 1732506015856 2024-11-25T03:40:15,859 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:40:15,859 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732506015859"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506015859"}]},"ts":"1732506015859"} 2024-11-25T03:40:15,862 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:40:15,863 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:40:15,863 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506015863"}]},"ts":"1732506015863"} 2024-11-25T03:40:15,866 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-25T03:40:15,866 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=251c13835aa3825db727b602ec851c98, ASSIGN}] 2024-11-25T03:40:15,868 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=251c13835aa3825db727b602ec851c98, ASSIGN 2024-11-25T03:40:15,869 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=251c13835aa3825db727b602ec851c98, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:40:16,020 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=251c13835aa3825db727b602ec851c98, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:40:16,023 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=251c13835aa3825db727b602ec851c98, ASSIGN because future has completed 2024-11-25T03:40:16,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 251c13835aa3825db727b602ec851c98, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:40:16,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T03:40:16,181 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:16,181 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 251c13835aa3825db727b602ec851c98, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:40:16,182 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,182 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:40:16,182 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,182 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,184 INFO [StoreOpener-251c13835aa3825db727b602ec851c98-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,186 INFO [StoreOpener-251c13835aa3825db727b602ec851c98-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 251c13835aa3825db727b602ec851c98 columnFamilyName testFamily 2024-11-25T03:40:16,186 DEBUG [StoreOpener-251c13835aa3825db727b602ec851c98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:40:16,186 INFO [StoreOpener-251c13835aa3825db727b602ec851c98-1 {}] regionserver.HStore(327): Store=251c13835aa3825db727b602ec851c98/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:40:16,187 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,187 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,188 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,190 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,193 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:40:16,194 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 251c13835aa3825db727b602ec851c98; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73917356, jitterRate=0.10145443677902222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:40:16,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:16,195 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 251c13835aa3825db727b602ec851c98: Running coprocessor pre-open hook at 1732506016182Writing region info on filesystem at 1732506016182Initializing all the Stores at 1732506016183 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506016183Cleaning up temporary data from old regions at 1732506016188 (+5 ms)Running coprocessor post-open hooks at 1732506016194 (+6 ms)Region opened successfully at 1732506016194 2024-11-25T03:40:16,196 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98., pid=30, masterSystemTime=1732506016176 2024-11-25T03:40:16,198 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:16,198 INFO [RS_OPEN_PRIORITY_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:16,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=251c13835aa3825db727b602ec851c98, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:40:16,201 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 251c13835aa3825db727b602ec851c98, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:40:16,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-25T03:40:16,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 251c13835aa3825db727b602ec851c98, server=38a5d2077268,36415,1732505907601 in 180 msec 2024-11-25T03:40:16,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-25T03:40:16,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=251c13835aa3825db727b602ec851c98, ASSIGN in 340 msec 2024-11-25T03:40:16,209 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:40:16,210 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506016210"}]},"ts":"1732506016210"} 2024-11-25T03:40:16,212 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-25T03:40:16,213 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:40:16,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.1810 sec 2024-11-25T03:40:17,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-25T03:40:17,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-25T03:40:18,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-25T03:40:18,066 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-25T03:40:18,066 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-25T03:40:18,066 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:40:18,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-25T03:40:18,071 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:40:18,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-25T03:40:18,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:40:18,083 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:40:18,083 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-25T03:40:18,086 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-25T03:40:18,088 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-25T03:40:18,089 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-25T03:40:18,090 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-25T03:40:18,662 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-25T03:40:21,684 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:40:24,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36415 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831412' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-25T03:40:24,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36415 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:56294 deadline: 1732506030100, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831412' does not exist 2024-11-25T03:40:24,103 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 38a5d2077268,36415,1732505907601 for closing scanner id = 5052773086642831412 for 251c13835aa3825db727b602ec851c98 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '5052773086642831412' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:40:24,110 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=444 (was 525), ProcessCount=11 (was 11), AvailableMemoryMB=6947 (was 7172) 2024-11-25T03:40:24,120 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=444, ProcessCount=11, AvailableMemoryMB=6946 2024-11-25T03:40:24,123 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:40:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-25T03:40:24,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:40:24,126 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:40:24,126 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-25T03:40:24,127 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:40:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:40:24,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741853_1029 (size=423) 2024-11-25T03:40:24,537 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cf6f8c32e09046510e7192401fcc9320, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977 2024-11-25T03:40:24,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741854_1030 (size=74) 2024-11-25T03:40:24,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-25T03:40:24,946 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:40:24,946 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing cf6f8c32e09046510e7192401fcc9320, disabling compactions & flushes 2024-11-25T03:40:24,946 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:24,946 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:24,946 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. after waiting 0 ms 2024-11-25T03:40:24,946 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:24,946 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:24,946 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for cf6f8c32e09046510e7192401fcc9320: Waiting for close lock at 1732506024946Disabling compacts and flushes for region at 1732506024946Disabling writes for close at 1732506024946Writing region close event to WAL at 1732506024946Closed at 1732506024946 2024-11-25T03:40:24,948 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:40:24,949 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732506024948"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506024948"}]},"ts":"1732506024948"} 2024-11-25T03:40:24,952 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:40:24,953 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:40:24,953 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506024953"}]},"ts":"1732506024953"} 2024-11-25T03:40:24,956 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-25T03:40:24,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cf6f8c32e09046510e7192401fcc9320, ASSIGN}] 2024-11-25T03:40:24,958 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cf6f8c32e09046510e7192401fcc9320, ASSIGN 2024-11-25T03:40:24,959 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cf6f8c32e09046510e7192401fcc9320, ASSIGN; state=OFFLINE, location=38a5d2077268,36415,1732505907601; forceNewPlan=false, retain=false 2024-11-25T03:40:25,109 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=cf6f8c32e09046510e7192401fcc9320, regionState=OPENING, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:40:25,112 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cf6f8c32e09046510e7192401fcc9320, ASSIGN because future has completed 2024-11-25T03:40:25,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf6f8c32e09046510e7192401fcc9320, server=38a5d2077268,36415,1732505907601}] 2024-11-25T03:40:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:40:25,270 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:25,271 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => cf6f8c32e09046510e7192401fcc9320, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:40:25,271 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,271 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:40:25,271 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,271 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,273 INFO [StoreOpener-cf6f8c32e09046510e7192401fcc9320-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,275 INFO [StoreOpener-cf6f8c32e09046510e7192401fcc9320-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf6f8c32e09046510e7192401fcc9320 columnFamilyName testFamily 2024-11-25T03:40:25,275 DEBUG [StoreOpener-cf6f8c32e09046510e7192401fcc9320-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:40:25,276 INFO [StoreOpener-cf6f8c32e09046510e7192401fcc9320-1 {}] regionserver.HStore(327): Store=cf6f8c32e09046510e7192401fcc9320/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:40:25,276 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,277 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,278 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,278 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,278 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,281 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,283 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:40:25,284 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened cf6f8c32e09046510e7192401fcc9320; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67469294, jitterRate=0.005370825529098511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:40:25,284 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:25,285 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for cf6f8c32e09046510e7192401fcc9320: Running coprocessor pre-open hook at 1732506025271Writing region info on filesystem at 1732506025271Initializing all the Stores at 1732506025273 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506025273Cleaning up temporary data from old regions at 1732506025278 (+5 ms)Running coprocessor post-open hooks at 1732506025284 (+6 ms)Region opened successfully at 1732506025285 (+1 ms) 2024-11-25T03:40:25,286 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., pid=33, masterSystemTime=1732506025265 2024-11-25T03:40:25,289 DEBUG [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:25,289 INFO [RS_OPEN_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:25,289 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=cf6f8c32e09046510e7192401fcc9320, regionState=OPEN, openSeqNum=2, regionLocation=38a5d2077268,36415,1732505907601 2024-11-25T03:40:25,292 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf6f8c32e09046510e7192401fcc9320, server=38a5d2077268,36415,1732505907601 because future has completed 2024-11-25T03:40:25,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-25T03:40:25,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure cf6f8c32e09046510e7192401fcc9320, server=38a5d2077268,36415,1732505907601 in 180 msec 2024-11-25T03:40:25,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-25T03:40:25,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=cf6f8c32e09046510e7192401fcc9320, ASSIGN in 339 msec 2024-11-25T03:40:25,298 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:40:25,298 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506025298"}]},"ts":"1732506025298"} 2024-11-25T03:40:25,301 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-25T03:40:25,302 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:40:25,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1790 sec 2024-11-25T03:40:25,634 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:40:27,153 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-25T03:40:27,153 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-25T03:40:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46447 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-25T03:40:27,156 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-25T03:40:27,156 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-25T03:40:27,156 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:40:27,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-25T03:40:27,161 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:40:27,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-25T03:40:27,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:40:27,173 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-25T03:40:27,173 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-25T03:40:27,176 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-25T03:40:27,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T03:40:28,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=38a5d2077268:36415 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-25T03:40:28,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-25T03:40:28,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2 from cache 2024-11-25T03:40:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-25T03:40:29,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2] 2024-11-25T03:40:29,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36415 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-25T03:40:30,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=38a5d2077268:36415 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=980ms, rpcTimeout=977ms 2024-11-25T03:40:30,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=980ms, rpcTimeout=977ms 2024-11-25T03:40:30,179 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., hostname=38a5d2077268,36415,1732505907601, seqNum=2 from cache 2024-11-25T03:40:30,189 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=221 (was 222), OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=416 (was 444), ProcessCount=11 (was 11), AvailableMemoryMB=6884 (was 6946) 2024-11-25T03:40:30,190 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:40:30,190 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:40:30,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:40:30,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:40:30,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:40:30,196 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T03:40:30,196 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:40:30,196 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:40:30,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:40:30,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:40:30,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:40:30,197 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T03:40:30,197 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1871265054, stopped=false 2024-11-25T03:40:30,197 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=38a5d2077268,46447,1732505906635 2024-11-25T03:40:30,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:40:30,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:40:30,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:40:30,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:40:30,199 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:40:30,199 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:40:30,199 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:40:30,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:40:30,200 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '38a5d2077268,36415,1732505907601' ***** 2024-11-25T03:40:30,200 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:40:30,200 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:40:30,201 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:40:30,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:40:30,201 INFO [RS:0;38a5d2077268:36415 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:40:30,201 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:40:30,201 INFO [RS:0;38a5d2077268:36415 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:40:30,201 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 0d484ec531b38a11fd8d9844c8403ad3 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 251c13835aa3825db727b602ec851c98 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for cf6f8c32e09046510e7192401fcc9320 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 091a1e050b8ecf2cad4c31bd0ba3a05c 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 5cc028e6e24e6f88d9a2913e959758ad 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 136205dd4c3406f1b0ed50672de492f7 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:40:30,202 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0d484ec531b38a11fd8d9844c8403ad3, disabling compactions & flushes 2024-11-25T03:40:30,202 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(959): stopping server 38a5d2077268,36415,1732505907601 2024-11-25T03:40:30,202 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:40:30,203 INFO [RS:0;38a5d2077268:36415 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:40:30,203 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:40:30,203 INFO [RS:0;38a5d2077268:36415 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;38a5d2077268:36415. 2024-11-25T03:40:30,203 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. after waiting 0 ms 2024-11-25T03:40:30,203 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:40:30,203 DEBUG [RS:0;38a5d2077268:36415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:40:30,203 DEBUG [RS:0;38a5d2077268:36415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:40:30,203 INFO [RS:0;38a5d2077268:36415 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:40:30,203 INFO [RS:0;38a5d2077268:36415 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:40:30,203 INFO [RS:0;38a5d2077268:36415 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:40:30,203 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:40:30,203 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-25T03:40:30,204 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:40:30,204 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:40:30,204 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:40:30,204 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:40:30,204 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:40:30,204 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 091a1e050b8ecf2cad4c31bd0ba3a05c, 0d484ec531b38a11fd8d9844c8403ad3, 136205dd4c3406f1b0ed50672de492f7, 1588230740, 1811ab78326ea2bfc621a27fb94cc3f7, 251c13835aa3825db727b602ec851c98, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, cf6f8c32e09046510e7192401fcc9320, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:30,207 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-25T03:40:30,207 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0d484ec531b38a11fd8d9844c8403ad3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:30,279 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3/.tmp/testFamily/157176417a1b4a4dbe35a63d8f9db566 is 61, key is row-0/testFamily:testQualifier/1732505996826/Put/seqid=0 2024-11-25T03:40:30,284 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/info/c00580130e53491f8a2fc33036b2b10d is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c./info:regioninfo/1732505971791/Put/seqid=0 2024-11-25T03:40:30,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741856_1032 (size=20564) 2024-11-25T03:40:30,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741855_1031 (size=5049) 2024-11-25T03:40:30,302 INFO [regionserver/38a5d2077268:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-25T03:40:30,302 INFO [regionserver/38a5d2077268:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-25T03:40:30,404 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 091a1e050b8ecf2cad4c31bd0ba3a05c, 0d484ec531b38a11fd8d9844c8403ad3, 136205dd4c3406f1b0ed50672de492f7, 1588230740, 1811ab78326ea2bfc621a27fb94cc3f7, 251c13835aa3825db727b602ec851c98, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, cf6f8c32e09046510e7192401fcc9320, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:30,604 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 091a1e050b8ecf2cad4c31bd0ba3a05c, 0d484ec531b38a11fd8d9844c8403ad3, 136205dd4c3406f1b0ed50672de492f7, 1588230740, 1811ab78326ea2bfc621a27fb94cc3f7, 251c13835aa3825db727b602ec851c98, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, cf6f8c32e09046510e7192401fcc9320, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:30,692 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/info/c00580130e53491f8a2fc33036b2b10d 2024-11-25T03:40:30,692 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3/.tmp/testFamily/157176417a1b4a4dbe35a63d8f9db566 2024-11-25T03:40:30,700 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-25T03:40:30,737 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3/.tmp/testFamily/157176417a1b4a4dbe35a63d8f9db566 as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3/testFamily/157176417a1b4a4dbe35a63d8f9db566 2024-11-25T03:40:30,746 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3/testFamily/157176417a1b4a4dbe35a63d8f9db566, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:30,752 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/ns/e7c05e2472eb462589515e053c995dbd is 43, key is default/ns:d/1732505970906/Put/seqid=0 2024-11-25T03:40:30,754 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0d484ec531b38a11fd8d9844c8403ad3 in 547ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:30,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741857_1033 (size=5153) 2024-11-25T03:40:30,761 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNextAsync/0d484ec531b38a11fd8d9844c8403ad3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:30,763 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:40:30,763 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0d484ec531b38a11fd8d9844c8403ad3: Waiting for close lock at 1732506030202Running coprocessor pre-close hooks at 1732506030202Disabling compacts and flushes for region at 1732506030202Disabling writes for close at 1732506030203 (+1 ms)Obtaining lock to block concurrent updates at 1732506030207 (+4 ms)Preparing flush snapshotting stores in 0d484ec531b38a11fd8d9844c8403ad3 at 1732506030207Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506030215 (+8 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. at 1732506030217 (+2 ms)Flushing 0d484ec531b38a11fd8d9844c8403ad3/testFamily: creating writer at 1732506030218 (+1 ms)Flushing 0d484ec531b38a11fd8d9844c8403ad3/testFamily: appending metadata at 1732506030272 (+54 ms)Flushing 0d484ec531b38a11fd8d9844c8403ad3/testFamily: closing flushed file at 1732506030275 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cb18534: reopening flushed file at 1732506030735 (+460 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0d484ec531b38a11fd8d9844c8403ad3 in 547ms, sequenceid=8, compaction requested=false at 1732506030754 (+19 ms)Writing region close event to WAL at 1732506030756 (+2 ms)Running coprocessor post-close hooks at 1732506030762 (+6 ms)Closed at 1732506030763 (+1 ms) 2024-11-25T03:40:30,764 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732505995796.0d484ec531b38a11fd8d9844c8403ad3. 2024-11-25T03:40:30,764 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 251c13835aa3825db727b602ec851c98, disabling compactions & flushes 2024-11-25T03:40:30,764 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:30,764 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:30,764 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. after waiting 0 ms 2024-11-25T03:40:30,764 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:30,764 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 251c13835aa3825db727b602ec851c98 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:30,773 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:40:30,785 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98/.tmp/testFamily/b3d341442b3d4807ab8574dfd928f9ae is 61, key is row-0/testFamily:testQualifier/1732506018075/Put/seqid=0 2024-11-25T03:40:30,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741858_1034 (size=5049) 2024-11-25T03:40:30,795 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98/.tmp/testFamily/b3d341442b3d4807ab8574dfd928f9ae 2024-11-25T03:40:30,803 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98/.tmp/testFamily/b3d341442b3d4807ab8574dfd928f9ae as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98/testFamily/b3d341442b3d4807ab8574dfd928f9ae 2024-11-25T03:40:30,804 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-25T03:40:30,805 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 091a1e050b8ecf2cad4c31bd0ba3a05c, 136205dd4c3406f1b0ed50672de492f7, 1588230740, 1811ab78326ea2bfc621a27fb94cc3f7, 251c13835aa3825db727b602ec851c98, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, cf6f8c32e09046510e7192401fcc9320, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:30,810 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98/testFamily/b3d341442b3d4807ab8574dfd928f9ae, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:30,812 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 251c13835aa3825db727b602ec851c98 in 47ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:30,816 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNext/251c13835aa3825db727b602ec851c98/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:30,817 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:30,817 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 251c13835aa3825db727b602ec851c98: Waiting for close lock at 1732506030764Running coprocessor pre-close hooks at 1732506030764Disabling compacts and flushes for region at 1732506030764Disabling writes for close at 1732506030764Obtaining lock to block concurrent updates at 1732506030764Preparing flush snapshotting stores in 251c13835aa3825db727b602ec851c98 at 1732506030764Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506030764Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. at 1732506030765 (+1 ms)Flushing 251c13835aa3825db727b602ec851c98/testFamily: creating writer at 1732506030765Flushing 251c13835aa3825db727b602ec851c98/testFamily: appending metadata at 1732506030784 (+19 ms)Flushing 251c13835aa3825db727b602ec851c98/testFamily: closing flushed file at 1732506030784Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54aa8a4a: reopening flushed file at 1732506030802 (+18 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 251c13835aa3825db727b602ec851c98 in 47ms, sequenceid=8, compaction requested=false at 1732506030812 (+10 ms)Writing region close event to WAL at 1732506030812Running coprocessor post-close hooks at 1732506030817 (+5 ms)Closed at 1732506030817 2024-11-25T03:40:30,817 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732506015031.251c13835aa3825db727b602ec851c98. 2024-11-25T03:40:30,817 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf6f8c32e09046510e7192401fcc9320, disabling compactions & flushes 2024-11-25T03:40:30,818 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:30,818 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:30,818 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. after waiting 0 ms 2024-11-25T03:40:30,818 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:30,818 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cf6f8c32e09046510e7192401fcc9320 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:30,835 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320/.tmp/testFamily/3761b843910d41fc8be5980e8f486af2 is 61, key is row-0/testFamily:testQualifier/1732506027165/Put/seqid=0 2024-11-25T03:40:30,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741859_1035 (size=5049) 2024-11-25T03:40:31,005 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 091a1e050b8ecf2cad4c31bd0ba3a05c, 136205dd4c3406f1b0ed50672de492f7, 1588230740, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, cf6f8c32e09046510e7192401fcc9320, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:31,158 DEBUG [master/38a5d2077268:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:40:31,159 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/ns/e7c05e2472eb462589515e053c995dbd 2024-11-25T03:40:31,185 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/table/7ec0a57eb9184e189f97fd7555a39747 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732505971812/Put/seqid=0 2024-11-25T03:40:31,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741860_1036 (size=6699) 2024-11-25T03:40:31,193 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/table/7ec0a57eb9184e189f97fd7555a39747 2024-11-25T03:40:31,203 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/info/c00580130e53491f8a2fc33036b2b10d as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/info/c00580130e53491f8a2fc33036b2b10d 2024-11-25T03:40:31,205 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-25T03:40:31,205 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1325): Online Regions={cf6f8c32e09046510e7192401fcc9320=testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., 091a1e050b8ecf2cad4c31bd0ba3a05c=testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c., 5cc028e6e24e6f88d9a2913e959758ad=testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad., d67a5cdf20dcb39f7da0b7f84a6b00e7=testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7., 1588230740=hbase:meta,,1.1588230740, 136205dd4c3406f1b0ed50672de492f7=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., 9f52f55338ad433d07df92f3d74b5266=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., 3d1f0936f96645ecc5d8c057f7aae55e=hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., 1811ab78326ea2bfc621a27fb94cc3f7=hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7.} 2024-11-25T03:40:31,206 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 091a1e050b8ecf2cad4c31bd0ba3a05c, 136205dd4c3406f1b0ed50672de492f7, 1588230740, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, cf6f8c32e09046510e7192401fcc9320, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:31,213 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/info/c00580130e53491f8a2fc33036b2b10d, entries=100, sequenceid=56, filesize=20.1 K 2024-11-25T03:40:31,214 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/ns/e7c05e2472eb462589515e053c995dbd as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/ns/e7c05e2472eb462589515e053c995dbd 2024-11-25T03:40:31,222 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/ns/e7c05e2472eb462589515e053c995dbd, entries=2, sequenceid=56, filesize=5.0 K 2024-11-25T03:40:31,223 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/.tmp/table/7ec0a57eb9184e189f97fd7555a39747 as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/table/7ec0a57eb9184e189f97fd7555a39747 2024-11-25T03:40:31,230 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/table/7ec0a57eb9184e189f97fd7555a39747, entries=20, sequenceid=56, filesize=6.5 K 2024-11-25T03:40:31,232 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1028ms, sequenceid=56, compaction requested=false 2024-11-25T03:40:31,238 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-25T03:40:31,238 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:40:31,239 INFO [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:40:31,239 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506030203Running coprocessor pre-close hooks at 1732506030204 (+1 ms)Disabling compacts and flushes for region at 1732506030204Disabling writes for close at 1732506030204Obtaining lock to block concurrent updates at 1732506030207 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1732506030207Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732506030215 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732506030217 (+2 ms)Flushing 1588230740/info: creating writer at 1732506030218 (+1 ms)Flushing 1588230740/info: appending metadata at 1732506030279 (+61 ms)Flushing 1588230740/info: closing flushed file at 1732506030279Flushing 1588230740/ns: creating writer at 1732506030735 (+456 ms)Flushing 1588230740/ns: appending metadata at 1732506030751 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732506030751Flushing 1588230740/table: creating writer at 1732506031168 (+417 ms)Flushing 1588230740/table: appending metadata at 1732506031184 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732506031184Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d36cd8e: reopening flushed file at 1732506031201 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d4beb52: reopening flushed file at 1732506031213 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33718189: reopening flushed file at 1732506031222 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1028ms, sequenceid=56, compaction requested=false at 1732506031232 (+10 ms)Writing region close event to WAL at 1732506031233 (+1 ms)Running coprocessor post-close hooks at 1732506031238 (+5 ms)Closed at 1732506031239 (+1 ms) 2024-11-25T03:40:31,239 DEBUG [RS_CLOSE_META-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:40:31,241 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320/.tmp/testFamily/3761b843910d41fc8be5980e8f486af2 2024-11-25T03:40:31,250 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320/.tmp/testFamily/3761b843910d41fc8be5980e8f486af2 as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320/testFamily/3761b843910d41fc8be5980e8f486af2 2024-11-25T03:40:31,258 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320/testFamily/3761b843910d41fc8be5980e8f486af2, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:31,259 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cf6f8c32e09046510e7192401fcc9320 in 441ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:31,264 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScannerAsync/cf6f8c32e09046510e7192401fcc9320/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:31,265 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:31,265 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf6f8c32e09046510e7192401fcc9320: Waiting for close lock at 1732506030817Running coprocessor pre-close hooks at 1732506030817Disabling compacts and flushes for region at 1732506030817Disabling writes for close at 1732506030818 (+1 ms)Obtaining lock to block concurrent updates at 1732506030818Preparing flush snapshotting stores in cf6f8c32e09046510e7192401fcc9320 at 1732506030818Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506030818Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. at 1732506030819 (+1 ms)Flushing cf6f8c32e09046510e7192401fcc9320/testFamily: creating writer at 1732506030819Flushing cf6f8c32e09046510e7192401fcc9320/testFamily: appending metadata at 1732506030835 (+16 ms)Flushing cf6f8c32e09046510e7192401fcc9320/testFamily: closing flushed file at 1732506030835Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e27d2da: reopening flushed file at 1732506031249 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cf6f8c32e09046510e7192401fcc9320 in 441ms, sequenceid=8, compaction requested=false at 1732506031259 (+10 ms)Writing region close event to WAL at 1732506031260 (+1 ms)Running coprocessor post-close hooks at 1732506031265 (+5 ms)Closed at 1732506031265 2024-11-25T03:40:31,265 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732506024122.cf6f8c32e09046510e7192401fcc9320. 2024-11-25T03:40:31,265 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 091a1e050b8ecf2cad4c31bd0ba3a05c, disabling compactions & flushes 2024-11-25T03:40:31,265 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:40:31,266 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:40:31,266 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. after waiting 0 ms 2024-11-25T03:40:31,266 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:40:31,266 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 091a1e050b8ecf2cad4c31bd0ba3a05c 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:31,284 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c/.tmp/testFamily/ecd3982daa52423e91403ef7f2937c76 is 61, key is row-0/testFamily:testQualifier/1732505972336/Put/seqid=0 2024-11-25T03:40:31,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741861_1037 (size=5049) 2024-11-25T03:40:31,406 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 091a1e050b8ecf2cad4c31bd0ba3a05c, 136205dd4c3406f1b0ed50672de492f7, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:31,606 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 091a1e050b8ecf2cad4c31bd0ba3a05c, 136205dd4c3406f1b0ed50672de492f7, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:31,691 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c/.tmp/testFamily/ecd3982daa52423e91403ef7f2937c76 2024-11-25T03:40:31,699 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c/.tmp/testFamily/ecd3982daa52423e91403ef7f2937c76 as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c/testFamily/ecd3982daa52423e91403ef7f2937c76 2024-11-25T03:40:31,705 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c/testFamily/ecd3982daa52423e91403ef7f2937c76, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:31,707 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 091a1e050b8ecf2cad4c31bd0ba3a05c in 440ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:31,711 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextExceptionAsync/091a1e050b8ecf2cad4c31bd0ba3a05c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:31,712 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:40:31,712 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 091a1e050b8ecf2cad4c31bd0ba3a05c: Waiting for close lock at 1732506031265Running coprocessor pre-close hooks at 1732506031265Disabling compacts and flushes for region at 1732506031265Disabling writes for close at 1732506031266 (+1 ms)Obtaining lock to block concurrent updates at 1732506031266Preparing flush snapshotting stores in 091a1e050b8ecf2cad4c31bd0ba3a05c at 1732506031266Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506031266Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. at 1732506031267 (+1 ms)Flushing 091a1e050b8ecf2cad4c31bd0ba3a05c/testFamily: creating writer at 1732506031267Flushing 091a1e050b8ecf2cad4c31bd0ba3a05c/testFamily: appending metadata at 1732506031283 (+16 ms)Flushing 091a1e050b8ecf2cad4c31bd0ba3a05c/testFamily: closing flushed file at 1732506031283Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@224fdf8c: reopening flushed file at 1732506031698 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 091a1e050b8ecf2cad4c31bd0ba3a05c in 440ms, sequenceid=8, compaction requested=false at 1732506031707 (+9 ms)Writing region close event to WAL at 1732506031707Running coprocessor post-close hooks at 1732506031712 (+5 ms)Closed at 1732506031712 2024-11-25T03:40:31,712 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732505971257.091a1e050b8ecf2cad4c31bd0ba3a05c. 2024-11-25T03:40:31,712 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5cc028e6e24e6f88d9a2913e959758ad, disabling compactions & flushes 2024-11-25T03:40:31,712 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:40:31,712 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:40:31,712 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. after waiting 0 ms 2024-11-25T03:40:31,712 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:40:31,712 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5cc028e6e24e6f88d9a2913e959758ad 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:31,729 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad/.tmp/testFamily/a52bb53fa11c4bac9709277ca4994424 is 61, key is row-0/testFamily:testQualifier/1732505989647/Put/seqid=0 2024-11-25T03:40:31,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741862_1038 (size=5049) 2024-11-25T03:40:31,806 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 136205dd4c3406f1b0ed50672de492f7, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:32,007 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 136205dd4c3406f1b0ed50672de492f7, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 5cc028e6e24e6f88d9a2913e959758ad, 9f52f55338ad433d07df92f3d74b5266, d67a5cdf20dcb39f7da0b7f84a6b00e7 2024-11-25T03:40:32,135 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad/.tmp/testFamily/a52bb53fa11c4bac9709277ca4994424 2024-11-25T03:40:32,144 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad/.tmp/testFamily/a52bb53fa11c4bac9709277ca4994424 as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad/testFamily/a52bb53fa11c4bac9709277ca4994424 2024-11-25T03:40:32,150 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad/testFamily/a52bb53fa11c4bac9709277ca4994424, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:32,151 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5cc028e6e24e6f88d9a2913e959758ad in 439ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:32,156 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testRetryOutOfOrderScannerNextException/5cc028e6e24e6f88d9a2913e959758ad/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:32,156 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:40:32,156 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5cc028e6e24e6f88d9a2913e959758ad: Waiting for close lock at 1732506031712Running coprocessor pre-close hooks at 1732506031712Disabling compacts and flushes for region at 1732506031712Disabling writes for close at 1732506031712Obtaining lock to block concurrent updates at 1732506031712Preparing flush snapshotting stores in 5cc028e6e24e6f88d9a2913e959758ad at 1732506031712Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506031712Flushing stores of testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. at 1732506031713 (+1 ms)Flushing 5cc028e6e24e6f88d9a2913e959758ad/testFamily: creating writer at 1732506031713Flushing 5cc028e6e24e6f88d9a2913e959758ad/testFamily: appending metadata at 1732506031729 (+16 ms)Flushing 5cc028e6e24e6f88d9a2913e959758ad/testFamily: closing flushed file at 1732506031729Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@329a82a3: reopening flushed file at 1732506032143 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5cc028e6e24e6f88d9a2913e959758ad in 439ms, sequenceid=8, compaction requested=false at 1732506032151 (+8 ms)Writing region close event to WAL at 1732506032152 (+1 ms)Running coprocessor post-close hooks at 1732506032156 (+4 ms)Closed at 1732506032156 2024-11-25T03:40:32,157 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732505986598.5cc028e6e24e6f88d9a2913e959758ad. 2024-11-25T03:40:32,157 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d67a5cdf20dcb39f7da0b7f84a6b00e7, disabling compactions & flushes 2024-11-25T03:40:32,157 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:32,157 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:32,157 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. after waiting 0 ms 2024-11-25T03:40:32,157 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:32,157 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d67a5cdf20dcb39f7da0b7f84a6b00e7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:32,173 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7/.tmp/testFamily/68acbf0a2d3d4bc487ad32cad25a29fc is 61, key is row-0/testFamily:testQualifier/1732506002916/Put/seqid=0 2024-11-25T03:40:32,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741863_1039 (size=5049) 2024-11-25T03:40:32,178 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7/.tmp/testFamily/68acbf0a2d3d4bc487ad32cad25a29fc 2024-11-25T03:40:32,187 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7/.tmp/testFamily/68acbf0a2d3d4bc487ad32cad25a29fc as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7/testFamily/68acbf0a2d3d4bc487ad32cad25a29fc 2024-11-25T03:40:32,195 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7/testFamily/68acbf0a2d3d4bc487ad32cad25a29fc, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:32,196 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d67a5cdf20dcb39f7da0b7f84a6b00e7 in 39ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:32,201 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnNext/d67a5cdf20dcb39f7da0b7f84a6b00e7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:32,202 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:32,202 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d67a5cdf20dcb39f7da0b7f84a6b00e7: Waiting for close lock at 1732506032157Running coprocessor pre-close hooks at 1732506032157Disabling compacts and flushes for region at 1732506032157Disabling writes for close at 1732506032157Obtaining lock to block concurrent updates at 1732506032157Preparing flush snapshotting stores in d67a5cdf20dcb39f7da0b7f84a6b00e7 at 1732506032157Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506032157Flushing stores of testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. at 1732506032158 (+1 ms)Flushing d67a5cdf20dcb39f7da0b7f84a6b00e7/testFamily: creating writer at 1732506032158Flushing d67a5cdf20dcb39f7da0b7f84a6b00e7/testFamily: appending metadata at 1732506032173 (+15 ms)Flushing d67a5cdf20dcb39f7da0b7f84a6b00e7/testFamily: closing flushed file at 1732506032173Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@363b9a3e: reopening flushed file at 1732506032185 (+12 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d67a5cdf20dcb39f7da0b7f84a6b00e7 in 39ms, sequenceid=8, compaction requested=false at 1732506032196 (+11 ms)Writing region close event to WAL at 1732506032197 (+1 ms)Running coprocessor post-close hooks at 1732506032202 (+5 ms)Closed at 1732506032202 2024-11-25T03:40:32,202 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732505999870.d67a5cdf20dcb39f7da0b7f84a6b00e7. 2024-11-25T03:40:32,202 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 136205dd4c3406f1b0ed50672de492f7, disabling compactions & flushes 2024-11-25T03:40:32,203 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:40:32,203 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:40:32,203 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. after waiting 0 ms 2024-11-25T03:40:32,203 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:40:32,203 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 136205dd4c3406f1b0ed50672de492f7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:32,207 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-25T03:40:32,207 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1325): Online Regions={136205dd4c3406f1b0ed50672de492f7=testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., 9f52f55338ad433d07df92f3d74b5266=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., 3d1f0936f96645ecc5d8c057f7aae55e=hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., 1811ab78326ea2bfc621a27fb94cc3f7=hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7.} 2024-11-25T03:40:32,207 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 136205dd4c3406f1b0ed50672de492f7, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:32,220 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7/.tmp/testFamily/38d58ab701334c71ba15f1f3d2514f19 is 61, key is row-0/testFamily:testQualifier/1732505992756/Put/seqid=0 2024-11-25T03:40:32,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741864_1040 (size=5049) 2024-11-25T03:40:32,407 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 136205dd4c3406f1b0ed50672de492f7, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:32,608 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 136205dd4c3406f1b0ed50672de492f7, 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:32,627 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7/.tmp/testFamily/38d58ab701334c71ba15f1f3d2514f19 2024-11-25T03:40:32,635 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7/.tmp/testFamily/38d58ab701334c71ba15f1f3d2514f19 as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7/testFamily/38d58ab701334c71ba15f1f3d2514f19 2024-11-25T03:40:32,642 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7/testFamily/38d58ab701334c71ba15f1f3d2514f19, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:32,643 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 136205dd4c3406f1b0ed50672de492f7 in 440ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:32,648 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/default/testNormalScanTimeoutOnOpenScanner/136205dd4c3406f1b0ed50672de492f7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:32,649 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:40:32,649 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 136205dd4c3406f1b0ed50672de492f7: Waiting for close lock at 1732506032202Running coprocessor pre-close hooks at 1732506032202Disabling compacts and flushes for region at 1732506032202Disabling writes for close at 1732506032203 (+1 ms)Obtaining lock to block concurrent updates at 1732506032203Preparing flush snapshotting stores in 136205dd4c3406f1b0ed50672de492f7 at 1732506032203Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506032203Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. at 1732506032204 (+1 ms)Flushing 136205dd4c3406f1b0ed50672de492f7/testFamily: creating writer at 1732506032204Flushing 136205dd4c3406f1b0ed50672de492f7/testFamily: appending metadata at 1732506032220 (+16 ms)Flushing 136205dd4c3406f1b0ed50672de492f7/testFamily: closing flushed file at 1732506032220Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1303e994: reopening flushed file at 1732506032634 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 136205dd4c3406f1b0ed50672de492f7 in 440ms, sequenceid=8, compaction requested=false at 1732506032643 (+9 ms)Writing region close event to WAL at 1732506032644 (+1 ms)Running coprocessor post-close hooks at 1732506032649 (+5 ms)Closed at 1732506032649 2024-11-25T03:40:32,649 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732505989706.136205dd4c3406f1b0ed50672de492f7. 2024-11-25T03:40:32,649 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9f52f55338ad433d07df92f3d74b5266, disabling compactions & flushes 2024-11-25T03:40:32,649 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:32,649 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:32,649 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. after waiting 0 ms 2024-11-25T03:40:32,650 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:32,650 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9f52f55338ad433d07df92f3d74b5266 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:32,673 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266/.tmp/testFamily/e3e4331d66ea4856869d522ba04ce65b is 61, key is row-0/testFamily:testQualifier/1732506008995/Put/seqid=0 2024-11-25T03:40:32,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741865_1041 (size=5049) 2024-11-25T03:40:32,808 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:33,008 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e, 9f52f55338ad433d07df92f3d74b5266 2024-11-25T03:40:33,079 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266/.tmp/testFamily/e3e4331d66ea4856869d522ba04ce65b 2024-11-25T03:40:33,088 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266/.tmp/testFamily/e3e4331d66ea4856869d522ba04ce65b as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266/testFamily/e3e4331d66ea4856869d522ba04ce65b 2024-11-25T03:40:33,096 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266/testFamily/e3e4331d66ea4856869d522ba04ce65b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:33,097 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9f52f55338ad433d07df92f3d74b5266 in 447ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:33,102 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/9f52f55338ad433d07df92f3d74b5266/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:33,102 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:33,102 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9f52f55338ad433d07df92f3d74b5266: Waiting for close lock at 1732506032649Running coprocessor pre-close hooks at 1732506032649Disabling compacts and flushes for region at 1732506032649Disabling writes for close at 1732506032650 (+1 ms)Obtaining lock to block concurrent updates at 1732506032650Preparing flush snapshotting stores in 9f52f55338ad433d07df92f3d74b5266 at 1732506032650Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506032650Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. at 1732506032651 (+1 ms)Flushing 9f52f55338ad433d07df92f3d74b5266/testFamily: creating writer at 1732506032651Flushing 9f52f55338ad433d07df92f3d74b5266/testFamily: appending metadata at 1732506032673 (+22 ms)Flushing 9f52f55338ad433d07df92f3d74b5266/testFamily: closing flushed file at 1732506032673Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@91dbee2: reopening flushed file at 1732506033087 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 9f52f55338ad433d07df92f3d74b5266 in 447ms, sequenceid=8, compaction requested=false at 1732506033097 (+10 ms)Writing region close event to WAL at 1732506033097Running coprocessor post-close hooks at 1732506033102 (+5 ms)Closed at 1732506033102 2024-11-25T03:40:33,103 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732506005958.9f52f55338ad433d07df92f3d74b5266. 2024-11-25T03:40:33,103 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3d1f0936f96645ecc5d8c057f7aae55e, disabling compactions & flushes 2024-11-25T03:40:33,103 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:40:33,103 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:40:33,103 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. after waiting 0 ms 2024-11-25T03:40:33,103 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:40:33,103 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3d1f0936f96645ecc5d8c057f7aae55e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:33,123 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e/.tmp/testFamily/46eed03dda7c418a95be0a7b83d2f2eb is 61, key is row-0/testFamily:testQualifier/1732505973468/Put/seqid=0 2024-11-25T03:40:33,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741866_1042 (size=5049) 2024-11-25T03:40:33,208 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T03:40:33,208 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1325): Online Regions={3d1f0936f96645ecc5d8c057f7aae55e=hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., 1811ab78326ea2bfc621a27fb94cc3f7=hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7.} 2024-11-25T03:40:33,209 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:40:33,409 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 1811ab78326ea2bfc621a27fb94cc3f7, 3d1f0936f96645ecc5d8c057f7aae55e 2024-11-25T03:40:33,529 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e/.tmp/testFamily/46eed03dda7c418a95be0a7b83d2f2eb 2024-11-25T03:40:33,538 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e/.tmp/testFamily/46eed03dda7c418a95be0a7b83d2f2eb as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e/testFamily/46eed03dda7c418a95be0a7b83d2f2eb 2024-11-25T03:40:33,546 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e/testFamily/46eed03dda7c418a95be0a7b83d2f2eb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:33,547 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3d1f0936f96645ecc5d8c057f7aae55e in 444ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:33,552 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnOpenScanner/3d1f0936f96645ecc5d8c057f7aae55e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:33,553 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:40:33,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3d1f0936f96645ecc5d8c057f7aae55e: Waiting for close lock at 1732506033103Running coprocessor pre-close hooks at 1732506033103Disabling compacts and flushes for region at 1732506033103Disabling writes for close at 1732506033103Obtaining lock to block concurrent updates at 1732506033103Preparing flush snapshotting stores in 3d1f0936f96645ecc5d8c057f7aae55e at 1732506033103Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506033103Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. at 1732506033104 (+1 ms)Flushing 3d1f0936f96645ecc5d8c057f7aae55e/testFamily: creating writer at 1732506033104Flushing 3d1f0936f96645ecc5d8c057f7aae55e/testFamily: appending metadata at 1732506033123 (+19 ms)Flushing 3d1f0936f96645ecc5d8c057f7aae55e/testFamily: closing flushed file at 1732506033123Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a2e72a9: reopening flushed file at 1732506033537 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3d1f0936f96645ecc5d8c057f7aae55e in 444ms, sequenceid=8, compaction requested=false at 1732506033547 (+10 ms)Writing region close event to WAL at 1732506033548 (+1 ms)Running coprocessor post-close hooks at 1732506033553 (+5 ms)Closed at 1732506033553 2024-11-25T03:40:33,553 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732505972435.3d1f0936f96645ecc5d8c057f7aae55e. 2024-11-25T03:40:33,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1811ab78326ea2bfc621a27fb94cc3f7, disabling compactions & flushes 2024-11-25T03:40:33,554 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:40:33,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:40:33,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. after waiting 0 ms 2024-11-25T03:40:33,554 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:40:33,554 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1811ab78326ea2bfc621a27fb94cc3f7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-25T03:40:33,571 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7/.tmp/testFamily/e9005c370b1b4a3a98f83636838e820d is 61, key is row-0/testFamily:testQualifier/1732505980548/Put/seqid=0 2024-11-25T03:40:33,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741867_1043 (size=5049) 2024-11-25T03:40:33,609 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:40:33,809 DEBUG [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1351): Waiting on 1811ab78326ea2bfc621a27fb94cc3f7 2024-11-25T03:40:33,976 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7/.tmp/testFamily/e9005c370b1b4a3a98f83636838e820d 2024-11-25T03:40:33,984 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7/.tmp/testFamily/e9005c370b1b4a3a98f83636838e820d as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7/testFamily/e9005c370b1b4a3a98f83636838e820d 2024-11-25T03:40:33,991 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7/testFamily/e9005c370b1b4a3a98f83636838e820d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-25T03:40:33,993 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1811ab78326ea2bfc621a27fb94cc3f7 in 439ms, sequenceid=8, compaction requested=false 2024-11-25T03:40:33,997 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/data/hbase/testMetaScanTimeoutOnNextAsync/1811ab78326ea2bfc621a27fb94cc3f7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-25T03:40:33,998 INFO [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:40:33,998 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1811ab78326ea2bfc621a27fb94cc3f7: Waiting for close lock at 1732506033554Running coprocessor pre-close hooks at 1732506033554Disabling compacts and flushes for region at 1732506033554Disabling writes for close at 1732506033554Obtaining lock to block concurrent updates at 1732506033554Preparing flush snapshotting stores in 1811ab78326ea2bfc621a27fb94cc3f7 at 1732506033554Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732506033554Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. at 1732506033555 (+1 ms)Flushing 1811ab78326ea2bfc621a27fb94cc3f7/testFamily: creating writer at 1732506033555Flushing 1811ab78326ea2bfc621a27fb94cc3f7/testFamily: appending metadata at 1732506033570 (+15 ms)Flushing 1811ab78326ea2bfc621a27fb94cc3f7/testFamily: closing flushed file at 1732506033570Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19134b6a: reopening flushed file at 1732506033983 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1811ab78326ea2bfc621a27fb94cc3f7 in 439ms, sequenceid=8, compaction requested=false at 1732506033993 (+10 ms)Writing region close event to WAL at 1732506033994 (+1 ms)Running coprocessor post-close hooks at 1732506033998 (+4 ms)Closed at 1732506033998 2024-11-25T03:40:33,998 DEBUG [RS_CLOSE_REGION-regionserver/38a5d2077268:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732505979521.1811ab78326ea2bfc621a27fb94cc3f7. 2024-11-25T03:40:34,010 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(976): stopping server 38a5d2077268,36415,1732505907601; all regions closed. 2024-11-25T03:40:34,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741834_1010 (size=19479) 2024-11-25T03:40:34,020 DEBUG [RS:0;38a5d2077268:36415 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/oldWALs 2024-11-25T03:40:34,020 INFO [RS:0;38a5d2077268:36415 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 38a5d2077268%2C36415%2C1732505907601.meta:.meta(num 1732505970716) 2024-11-25T03:40:34,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741833_1009 (size=20031) 2024-11-25T03:40:34,023 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/WALs/38a5d2077268,36415,1732505907601/38a5d2077268%2C36415%2C1732505907601.1732505970337 not finished, retry = 0 2024-11-25T03:40:34,127 DEBUG [RS:0;38a5d2077268:36415 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/oldWALs 2024-11-25T03:40:34,127 INFO [RS:0;38a5d2077268:36415 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 38a5d2077268%2C36415%2C1732505907601:(num 1732505970337) 2024-11-25T03:40:34,127 DEBUG [RS:0;38a5d2077268:36415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:40:34,127 INFO [RS:0;38a5d2077268:36415 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:40:34,127 INFO [RS:0;38a5d2077268:36415 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:40:34,127 INFO [RS:0;38a5d2077268:36415 {}] hbase.ChoreService(370): Chore service for: regionserver/38a5d2077268:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:40:34,128 INFO [RS:0;38a5d2077268:36415 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:40:34,128 INFO [regionserver/38a5d2077268:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:40:34,129 INFO [RS:0;38a5d2077268:36415 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36415 2024-11-25T03:40:34,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/38a5d2077268,36415,1732505907601 2024-11-25T03:40:34,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:40:34,132 INFO [RS:0;38a5d2077268:36415 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:40:34,134 INFO [regionserver/38a5d2077268:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:40:34,135 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [38a5d2077268,36415,1732505907601] 2024-11-25T03:40:34,140 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/38a5d2077268,36415,1732505907601 already deleted, retry=false 2024-11-25T03:40:34,140 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 38a5d2077268,36415,1732505907601 expired; onlineServers=0 2024-11-25T03:40:34,140 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '38a5d2077268,46447,1732505906635' ***** 2024-11-25T03:40:34,140 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T03:40:34,140 INFO [M:0;38a5d2077268:46447 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:40:34,141 INFO [M:0;38a5d2077268:46447 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:40:34,141 DEBUG [M:0;38a5d2077268:46447 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T03:40:34,141 DEBUG [M:0;38a5d2077268:46447 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T03:40:34,141 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T03:40:34,141 DEBUG [master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732505910124 {}] cleaner.HFileCleaner(306): Exit Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.large.0-1732505910124,5,FailOnTimeoutGroup] 2024-11-25T03:40:34,141 DEBUG [master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732505910129 {}] cleaner.HFileCleaner(306): Exit Thread[master/38a5d2077268:0:becomeActiveMaster-HFileCleaner.small.0-1732505910129,5,FailOnTimeoutGroup] 2024-11-25T03:40:34,141 INFO [M:0;38a5d2077268:46447 {}] hbase.ChoreService(370): Chore service for: master/38a5d2077268:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T03:40:34,141 INFO [M:0;38a5d2077268:46447 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:40:34,142 DEBUG [M:0;38a5d2077268:46447 {}] master.HMaster(1795): Stopping service threads 2024-11-25T03:40:34,142 INFO [M:0;38a5d2077268:46447 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T03:40:34,142 INFO [M:0;38a5d2077268:46447 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:40:34,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T03:40:34,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:40:34,142 INFO [M:0;38a5d2077268:46447 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:40:34,143 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T03:40:34,143 DEBUG [M:0;38a5d2077268:46447 {}] zookeeper.ZKUtil(347): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T03:40:34,143 WARN [M:0;38a5d2077268:46447 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T03:40:34,144 INFO [M:0;38a5d2077268:46447 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/.lastflushedseqids 2024-11-25T03:40:34,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741868_1044 (size=460) 2024-11-25T03:40:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:40:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36415-0x1014d3414b00001, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:40:34,235 INFO [RS:0;38a5d2077268:36415 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:40:34,236 INFO [RS:0;38a5d2077268:36415 {}] regionserver.HRegionServer(1031): Exiting; stopping=38a5d2077268,36415,1732505907601; zookeeper connection closed. 2024-11-25T03:40:34,236 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@30adf28c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@30adf28c 2024-11-25T03:40:34,237 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-25T03:40:34,557 INFO [M:0;38a5d2077268:46447 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:40:34,557 INFO [M:0;38a5d2077268:46447 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T03:40:34,557 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:40:34,557 INFO [M:0;38a5d2077268:46447 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:40:34,557 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:40:34,558 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:40:34,558 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:40:34,558 INFO [M:0;38a5d2077268:46447 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.04 KB heapSize=191.02 KB 2024-11-25T03:40:34,576 DEBUG [M:0;38a5d2077268:46447 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4697001d44f478fa24a36dba488b523 is 82, key is hbase:meta,,1/info:regioninfo/1732505970814/Put/seqid=0 2024-11-25T03:40:34,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741869_1045 (size=5672) 2024-11-25T03:40:34,984 INFO [M:0;38a5d2077268:46447 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4697001d44f478fa24a36dba488b523 2024-11-25T03:40:35,008 DEBUG [M:0;38a5d2077268:46447 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59502082253145ecbf07b45b6d04832c is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732505971822/Put/seqid=0 2024-11-25T03:40:35,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741870_1046 (size=14065) 2024-11-25T03:40:35,414 INFO [M:0;38a5d2077268:46447 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59502082253145ecbf07b45b6d04832c 2024-11-25T03:40:35,421 INFO [M:0;38a5d2077268:46447 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 59502082253145ecbf07b45b6d04832c 2024-11-25T03:40:35,437 DEBUG [M:0;38a5d2077268:46447 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8fb394766b854e8c9b3ff148628643eb is 69, key is 38a5d2077268,36415,1732505907601/rs:state/1732505940117/Put/seqid=0 2024-11-25T03:40:35,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741871_1047 (size=5156) 2024-11-25T03:40:35,842 INFO [M:0;38a5d2077268:46447 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8fb394766b854e8c9b3ff148628643eb 2024-11-25T03:40:35,850 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a4697001d44f478fa24a36dba488b523 as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a4697001d44f478fa24a36dba488b523 2024-11-25T03:40:35,856 INFO [M:0;38a5d2077268:46447 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a4697001d44f478fa24a36dba488b523, entries=8, sequenceid=319, filesize=5.5 K 2024-11-25T03:40:35,857 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/59502082253145ecbf07b45b6d04832c as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/59502082253145ecbf07b45b6d04832c 2024-11-25T03:40:35,862 INFO [M:0;38a5d2077268:46447 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 59502082253145ecbf07b45b6d04832c 2024-11-25T03:40:35,862 INFO [M:0;38a5d2077268:46447 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/59502082253145ecbf07b45b6d04832c, entries=33, sequenceid=319, filesize=13.7 K 2024-11-25T03:40:35,863 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8fb394766b854e8c9b3ff148628643eb as hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8fb394766b854e8c9b3ff148628643eb 2024-11-25T03:40:35,868 INFO [M:0;38a5d2077268:46447 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45233/user/jenkins/test-data/7b87da39-d464-1d42-5104-4ee6c1ef0977/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8fb394766b854e8c9b3ff148628643eb, entries=1, sequenceid=319, filesize=5.0 K 2024-11-25T03:40:35,869 INFO [M:0;38a5d2077268:46447 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.04 KB/166955, heapSize ~190.72 KB/195296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1311ms, sequenceid=319, compaction requested=false 2024-11-25T03:40:35,871 INFO [M:0;38a5d2077268:46447 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:40:35,871 DEBUG [M:0;38a5d2077268:46447 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732506034557Disabling compacts and flushes for region at 1732506034557Disabling writes for close at 1732506034558 (+1 ms)Obtaining lock to block concurrent updates at 1732506034558Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732506034558Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166955, getHeapSize=195536, getOffHeapSize=0, getCellsCount=366 at 1732506034558Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732506034559 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732506034559Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732506034576 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732506034576Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732506034991 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732506035008 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732506035008Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732506035421 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732506035436 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732506035436Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65fb0a62: reopening flushed file at 1732506035849 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73c427ae: reopening flushed file at 1732506035856 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d16a8d0: reopening flushed file at 1732506035862 (+6 ms)Finished flush of dataSize ~163.04 KB/166955, heapSize ~190.72 KB/195296, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1311ms, sequenceid=319, compaction requested=false at 1732506035869 (+7 ms)Writing region close event to WAL at 1732506035871 (+2 ms)Closed at 1732506035871 2024-11-25T03:40:35,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42577 is added to blk_1073741830_1006 (size=188297) 2024-11-25T03:40:35,874 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:40:35,875 INFO [M:0;38a5d2077268:46447 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T03:40:35,875 INFO [M:0;38a5d2077268:46447 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46447 2024-11-25T03:40:35,875 INFO [M:0;38a5d2077268:46447 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:40:35,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:40:35,977 INFO [M:0;38a5d2077268:46447 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:40:35,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46447-0x1014d3414b00000, quorum=127.0.0.1:58447, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:40:35,982 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@429b6cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:40:35,984 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:40:35,984 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:40:35,984 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:40:35,985 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/hadoop.log.dir/,STOPPED} 2024-11-25T03:40:35,988 WARN [BP-493315070-172.17.0.2-1732505903679 heartbeating to localhost/127.0.0.1:45233 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T03:40:35,988 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T03:40:35,988 WARN [BP-493315070-172.17.0.2-1732505903679 heartbeating to localhost/127.0.0.1:45233 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-493315070-172.17.0.2-1732505903679 (Datanode Uuid 5a8be03a-2f9b-4613-8646-d792372e733b) service to localhost/127.0.0.1:45233 2024-11-25T03:40:35,988 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T03:40:35,989 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/cluster_052b3b17-9ee8-7e2f-08fb-e0d632542370/data/data1/current/BP-493315070-172.17.0.2-1732505903679 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:40:35,989 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/cluster_052b3b17-9ee8-7e2f-08fb-e0d632542370/data/data2/current/BP-493315070-172.17.0.2-1732505903679 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:40:35,990 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T03:40:35,996 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:40:35,996 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:40:35,996 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:40:35,997 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:40:35,997 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45aae3d9-5701-fd8a-3ba2-15b883529318/hadoop.log.dir/,STOPPED} 2024-11-25T03:40:36,004 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T03:40:36,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down