2024-11-22 16:55:33,905 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-22 16:55:33,937 main DEBUG Took 0.029784 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 16:55:33,938 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 16:55:33,938 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 16:55:33,939 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 16:55:33,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,951 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 16:55:33,967 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,970 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,971 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,973 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,974 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,975 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,977 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,978 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,979 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,980 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,980 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,981 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,982 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 16:55:33,983 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,984 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 16:55:33,986 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 16:55:33,987 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 16:55:33,990 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 16:55:33,990 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 16:55:33,991 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 16:55:33,992 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 16:55:34,004 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 16:55:34,007 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 16:55:34,009 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 16:55:34,010 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 16:55:34,010 main DEBUG createAppenders(={Console}) 2024-11-22 16:55:34,011 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-22 16:55:34,012 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-22 16:55:34,012 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-22 16:55:34,013 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 16:55:34,013 main DEBUG OutputStream closed 2024-11-22 16:55:34,013 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 16:55:34,014 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 16:55:34,014 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-22 16:55:34,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 16:55:34,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 16:55:34,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 16:55:34,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 16:55:34,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 16:55:34,111 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 16:55:34,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 16:55:34,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 16:55:34,112 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 16:55:34,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 16:55:34,113 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 16:55:34,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 16:55:34,114 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 16:55:34,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 16:55:34,115 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 16:55:34,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 16:55:34,116 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 16:55:34,117 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 16:55:34,120 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 16:55:34,121 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-22 16:55:34,121 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 16:55:34,122 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-22T16:55:34,137 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-22 16:55:34,140 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 16:55:34,140 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T16:55:34,441 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547 2024-11-22T16:55:34,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-22T16:55:34,483 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/cluster_a6831cb5-71e9-f3e9-7e8c-861cbb1dc9c4, deleteOnExit=true 2024-11-22T16:55:34,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-22T16:55:34,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/test.cache.data in system properties and HBase conf 2024-11-22T16:55:34,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/hadoop.tmp.dir in system properties and HBase conf 2024-11-22T16:55:34,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/hadoop.log.dir in system properties and HBase conf 2024-11-22T16:55:34,486 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-22T16:55:34,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-22T16:55:34,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-22T16:55:34,611 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T16:55:34,744 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T16:55:34,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-22T16:55:34,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-22T16:55:34,751 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-22T16:55:34,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T16:55:34,752 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-22T16:55:34,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-22T16:55:34,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-22T16:55:34,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T16:55:34,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-22T16:55:34,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/nfs.dump.dir in system properties and HBase conf 2024-11-22T16:55:34,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/java.io.tmpdir in system properties and HBase conf 2024-11-22T16:55:34,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-22T16:55:34,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-22T16:55:34,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-22T16:55:35,711 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-22T16:55:35,797 INFO [Time-limited test {}] log.Log(170): Logging initialized @2798ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-22T16:55:35,883 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T16:55:35,956 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T16:55:35,990 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T16:55:35,991 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T16:55:35,993 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-22T16:55:36,012 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T16:55:36,016 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/hadoop.log.dir/,AVAILABLE} 2024-11-22T16:55:36,017 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T16:55:36,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/java.io.tmpdir/jetty-localhost-37679-hadoop-hdfs-3_4_1-tests_jar-_-any-5423217761282289269/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T16:55:36,280 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:37679} 2024-11-22T16:55:36,280 INFO [Time-limited test {}] server.Server(415): Started @3283ms 2024-11-22T16:55:36,736 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-22T16:55:36,743 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-22T16:55:36,744 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-22T16:55:36,745 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-22T16:55:36,745 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-22T16:55:36,747 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/hadoop.log.dir/,AVAILABLE} 2024-11-22T16:55:36,748 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-22T16:55:36,914 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/java.io.tmpdir/jetty-localhost-43505-hadoop-hdfs-3_4_1-tests_jar-_-any-12456183005589021984/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T16:55:36,915 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:43505} 2024-11-22T16:55:36,915 INFO [Time-limited test {}] server.Server(415): Started @3918ms 2024-11-22T16:55:36,976 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-22T16:55:37,486 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/cluster_a6831cb5-71e9-f3e9-7e8c-861cbb1dc9c4/data/data2/current/BP-982762458-172.17.0.2-1732294535442/current, will proceed with Du for space computation calculation, 2024-11-22T16:55:37,486 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/cluster_a6831cb5-71e9-f3e9-7e8c-861cbb1dc9c4/data/data1/current/BP-982762458-172.17.0.2-1732294535442/current, will proceed with Du for space computation calculation, 2024-11-22T16:55:37,524 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-22T16:55:37,580 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa6c596a10f412d92 with lease ID 0x9e65e617f01b56c2: Processing first storage report for DS-44b0140a-53fc-4308-ae89-5803ba993d1e from datanode DatanodeRegistration(127.0.0.1:33941, datanodeUuid=be375568-4cf4-4f76-bb45-a1c00c422d38, infoPort=45349, infoSecurePort=0, ipcPort=39347, storageInfo=lv=-57;cid=testClusterID;nsid=70204564;c=1732294535442) 2024-11-22T16:55:37,581 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa6c596a10f412d92 with lease ID 0x9e65e617f01b56c2: from storage DS-44b0140a-53fc-4308-ae89-5803ba993d1e node DatanodeRegistration(127.0.0.1:33941, datanodeUuid=be375568-4cf4-4f76-bb45-a1c00c422d38, infoPort=45349, infoSecurePort=0, ipcPort=39347, storageInfo=lv=-57;cid=testClusterID;nsid=70204564;c=1732294535442), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-22T16:55:37,581 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa6c596a10f412d92 with lease ID 0x9e65e617f01b56c2: Processing first storage report for DS-fe1aead9-0a3d-4bb3-9936-a1c1d1dc375f from datanode DatanodeRegistration(127.0.0.1:33941, datanodeUuid=be375568-4cf4-4f76-bb45-a1c00c422d38, infoPort=45349, infoSecurePort=0, ipcPort=39347, storageInfo=lv=-57;cid=testClusterID;nsid=70204564;c=1732294535442) 2024-11-22T16:55:37,582 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa6c596a10f412d92 with lease ID 0x9e65e617f01b56c2: from storage DS-fe1aead9-0a3d-4bb3-9936-a1c1d1dc375f node DatanodeRegistration(127.0.0.1:33941, datanodeUuid=be375568-4cf4-4f76-bb45-a1c00c422d38, infoPort=45349, infoSecurePort=0, ipcPort=39347, storageInfo=lv=-57;cid=testClusterID;nsid=70204564;c=1732294535442), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-22T16:55:37,585 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547 2024-11-22T16:55:37,667 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/cluster_a6831cb5-71e9-f3e9-7e8c-861cbb1dc9c4/zookeeper_0, clientPort=57407, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/cluster_a6831cb5-71e9-f3e9-7e8c-861cbb1dc9c4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/cluster_a6831cb5-71e9-f3e9-7e8c-861cbb1dc9c4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T16:55:37,677 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57407 2024-11-22T16:55:37,691 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:55:37,695 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:55:38,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741825_1001 (size=7) 2024-11-22T16:55:38,433 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d with version=8 2024-11-22T16:55:38,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/hbase-staging 2024-11-22T16:55:38,535 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T16:55:38,785 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4618c6346924:0 server-side Connection retries=9 2024-11-22T16:55:38,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:55:38,796 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:55:38,800 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:55:38,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:55:38,801 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:55:38,939 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T16:55:39,006 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T16:55:39,015 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T16:55:39,019 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:55:39,047 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5778 (auto-detected) 2024-11-22T16:55:39,048 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T16:55:39,067 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38813 2024-11-22T16:55:39,092 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38813 connecting to ZooKeeper ensemble=127.0.0.1:57407 2024-11-22T16:55:39,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388130x0, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:55:39,130 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38813-0x101409adc3d0000 connected 2024-11-22T16:55:39,156 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:55:39,158 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:55:39,169 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:55:39,173 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d, hbase.cluster.distributed=false 2024-11-22T16:55:39,195 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:55:39,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-22T16:55:39,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38813 2024-11-22T16:55:39,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38813 2024-11-22T16:55:39,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-22T16:55:39,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38813 2024-11-22T16:55:39,317 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4618c6346924:0 server-side Connection retries=9 2024-11-22T16:55:39,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:55:39,319 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T16:55:39,319 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T16:55:39,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T16:55:39,320 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T16:55:39,322 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T16:55:39,325 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T16:55:39,326 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35523 2024-11-22T16:55:39,328 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35523 connecting to ZooKeeper ensemble=127.0.0.1:57407 2024-11-22T16:55:39,329 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:55:39,332 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:55:39,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355230x0, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T16:55:39,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:355230x0, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:55:39,341 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35523-0x101409adc3d0001 connected 2024-11-22T16:55:39,345 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T16:55:39,353 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T16:55:39,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T16:55:39,361 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T16:55:39,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35523 2024-11-22T16:55:39,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35523 2024-11-22T16:55:39,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35523 2024-11-22T16:55:39,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35523 2024-11-22T16:55:39,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35523 2024-11-22T16:55:39,384 DEBUG [M:0;4618c6346924:38813 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4618c6346924:38813 2024-11-22T16:55:39,385 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4618c6346924,38813,1732294538589 2024-11-22T16:55:39,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:55:39,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:55:39,394 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4618c6346924,38813,1732294538589 2024-11-22T16:55:39,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:39,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T16:55:39,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:39,416 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T16:55:39,418 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4618c6346924,38813,1732294538589 from backup master directory 2024-11-22T16:55:39,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4618c6346924,38813,1732294538589 2024-11-22T16:55:39,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:55:39,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T16:55:39,421 WARN [master/4618c6346924:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:55:39,421 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4618c6346924,38813,1732294538589 2024-11-22T16:55:39,424 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T16:55:39,425 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T16:55:39,483 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/hbase.id] with ID: 5de1f6e5-c0b9-4345-b601-54e90cd293c4 2024-11-22T16:55:39,483 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/.tmp/hbase.id 2024-11-22T16:55:39,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741826_1002 (size=42) 2024-11-22T16:55:39,896 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/.tmp/hbase.id]:[hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/hbase.id] 2024-11-22T16:55:39,947 INFO [master/4618c6346924:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-22T16:55:39,952 INFO [master/4618c6346924:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T16:55:39,975 INFO [master/4618c6346924:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-22T16:55:39,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:39,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:39,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741827_1003 (size=196) 2024-11-22T16:55:40,018 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:55:40,022 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T16:55:40,046 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:55:40,051 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:55:40,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741828_1004 (size=1189) 2024-11-22T16:55:40,505 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store 2024-11-22T16:55:40,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741829_1005 (size=34) 2024-11-22T16:55:40,932 INFO [master/4618c6346924:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T16:55:40,935 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:55:40,936 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T16:55:40,936 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:55:40,936 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:55:40,938 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T16:55:40,938 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:55:40,938 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:55:40,939 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732294540936Disabling compacts and flushes for region at 1732294540936Disabling writes for close at 1732294540938 (+2 ms)Writing region close event to WAL at 1732294540938Closed at 1732294540938 2024-11-22T16:55:40,942 WARN [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/.initializing 2024-11-22T16:55:40,942 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/WALs/4618c6346924,38813,1732294538589 2024-11-22T16:55:40,951 INFO [master/4618c6346924:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:55:40,968 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4618c6346924%2C38813%2C1732294538589, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/WALs/4618c6346924,38813,1732294538589, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/oldWALs, maxLogs=10 2024-11-22T16:55:40,996 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/WALs/4618c6346924,38813,1732294538589/4618c6346924%2C38813%2C1732294538589.1732294540973, exclude list is [], retry=0 2024-11-22T16:55:41,017 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33941,DS-44b0140a-53fc-4308-ae89-5803ba993d1e,DISK] 2024-11-22T16:55:41,022 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-22T16:55:41,073 INFO [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/WALs/4618c6346924,38813,1732294538589/4618c6346924%2C38813%2C1732294538589.1732294540973 2024-11-22T16:55:41,074 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-22T16:55:41,075 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:55:41,076 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:55:41,079 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,080 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,123 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T16:55:41,166 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:55:41,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,176 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T16:55:41,176 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:55:41,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,182 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T16:55:41,182 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:55:41,184 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T16:55:41,188 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,189 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:55:41,190 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,195 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,196 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,204 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,204 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,209 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:55:41,214 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T16:55:41,220 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:55:41,221 INFO [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71031627, jitterRate=0.058453723788261414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:55:41,230 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732294541093Initializing all the Stores at 1732294541096 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732294541097 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294541098 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294541098Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294541098Cleaning up temporary data from old regions at 1732294541205 (+107 ms)Region opened successfully at 1732294541229 (+24 ms) 2024-11-22T16:55:41,231 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T16:55:41,272 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24889351, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:55:41,309 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T16:55:41,321 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T16:55:41,321 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T16:55:41,324 INFO [master/4618c6346924:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T16:55:41,326 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-22T16:55:41,331 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-22T16:55:41,331 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T16:55:41,363 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T16:55:41,374 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T16:55:41,376 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T16:55:41,379 INFO [master/4618c6346924:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T16:55:41,381 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T16:55:41,383 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T16:55:41,385 INFO [master/4618c6346924:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T16:55:41,390 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T16:55:41,391 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T16:55:41,393 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T16:55:41,396 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T16:55:41,418 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T16:55:41,420 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T16:55:41,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:55:41,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T16:55:41,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:41,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:41,429 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4618c6346924,38813,1732294538589, sessionid=0x101409adc3d0000, setting cluster-up flag (Was=false) 2024-11-22T16:55:41,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:41,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:41,452 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T16:55:41,454 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4618c6346924,38813,1732294538589 2024-11-22T16:55:41,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:41,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:55:41,468 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T16:55:41,470 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4618c6346924,38813,1732294538589 2024-11-22T16:55:41,479 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T16:55:41,577 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(746): ClusterId : 5de1f6e5-c0b9-4345-b601-54e90cd293c4 2024-11-22T16:55:41,580 DEBUG [RS:0;4618c6346924:35523 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T16:55:41,585 DEBUG [RS:0;4618c6346924:35523 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T16:55:41,585 DEBUG [RS:0;4618c6346924:35523 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T16:55:41,587 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T16:55:41,588 DEBUG [RS:0;4618c6346924:35523 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T16:55:41,589 DEBUG [RS:0;4618c6346924:35523 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d359883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4618c6346924/172.17.0.2:0 2024-11-22T16:55:41,597 INFO [master/4618c6346924:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T16:55:41,603 INFO [master/4618c6346924:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T16:55:41,604 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4618c6346924:35523 2024-11-22T16:55:41,608 INFO [RS:0;4618c6346924:35523 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T16:55:41,608 INFO [RS:0;4618c6346924:35523 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T16:55:41,608 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T16:55:41,611 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(2659): reportForDuty to master=4618c6346924,38813,1732294538589 with port=35523, startcode=1732294539277 2024-11-22T16:55:41,609 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4618c6346924,38813,1732294538589 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T16:55:41,617 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:55:41,617 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:55:41,617 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:55:41,618 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4618c6346924:0, corePoolSize=5, maxPoolSize=5 2024-11-22T16:55:41,618 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4618c6346924:0, corePoolSize=10, maxPoolSize=10 2024-11-22T16:55:41,618 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,618 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:55:41,619 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,620 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732294571620 2024-11-22T16:55:41,622 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T16:55:41,624 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T16:55:41,624 DEBUG [RS:0;4618c6346924:35523 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T16:55:41,626 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:55:41,626 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T16:55:41,627 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T16:55:41,628 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T16:55:41,628 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T16:55:41,628 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T16:55:41,629 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,633 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T16:55:41,635 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T16:55:41,635 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,635 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T16:55:41,636 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T16:55:41,638 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T16:55:41,638 INFO [master/4618c6346924:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T16:55:41,641 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732294541640,5,FailOnTimeoutGroup] 2024-11-22T16:55:41,644 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732294541641,5,FailOnTimeoutGroup] 2024-11-22T16:55:41,644 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,645 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T16:55:41,646 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,646 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741831_1007 (size=1321) 2024-11-22T16:55:41,654 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T16:55:41,655 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:55:41,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741832_1008 (size=32) 2024-11-22T16:55:41,666 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46827, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T16:55:41,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:55:41,672 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38813 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4618c6346924,35523,1732294539277 2024-11-22T16:55:41,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T16:55:41,675 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38813 {}] master.ServerManager(517): Registering regionserver=4618c6346924,35523,1732294539277 2024-11-22T16:55:41,676 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T16:55:41,676 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:55:41,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T16:55:41,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T16:55:41,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:55:41,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T16:55:41,684 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T16:55:41,684 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,685 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:55:41,686 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T16:55:41,689 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T16:55:41,689 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:55:41,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:55:41,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T16:55:41,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740 2024-11-22T16:55:41,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740 2024-11-22T16:55:41,692 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:55:41,692 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35361 2024-11-22T16:55:41,693 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T16:55:41,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T16:55:41,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T16:55:41,697 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:55:41,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:55:41,698 DEBUG [RS:0;4618c6346924:35523 {}] zookeeper.ZKUtil(111): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4618c6346924,35523,1732294539277 2024-11-22T16:55:41,699 WARN [RS:0;4618c6346924:35523 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T16:55:41,699 INFO [RS:0;4618c6346924:35523 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:55:41,699 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/WALs/4618c6346924,35523,1732294539277 2024-11-22T16:55:41,699 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T16:55:41,703 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4618c6346924,35523,1732294539277] 2024-11-22T16:55:41,705 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:55:41,706 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73368787, jitterRate=0.0932801216840744}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:55:41,709 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732294541671Initializing all the Stores at 1732294541673 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732294541673Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732294541673Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294541673Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732294541673Cleaning up temporary data from old regions at 1732294541696 (+23 ms)Region opened successfully at 1732294541709 (+13 ms) 2024-11-22T16:55:41,709 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T16:55:41,709 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T16:55:41,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T16:55:41,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T16:55:41,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T16:55:41,711 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T16:55:41,712 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732294541709Disabling compacts and flushes for region at 1732294541709Disabling writes for close at 1732294541710 (+1 ms)Writing region close event to WAL at 1732294541711 (+1 ms)Closed at 1732294541711 2024-11-22T16:55:41,716 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:55:41,716 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T16:55:41,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T16:55:41,727 INFO [RS:0;4618c6346924:35523 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T16:55:41,731 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T16:55:41,734 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T16:55:41,742 INFO [RS:0;4618c6346924:35523 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T16:55:41,747 INFO [RS:0;4618c6346924:35523 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T16:55:41,747 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,748 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T16:55:41,754 INFO [RS:0;4618c6346924:35523 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T16:55:41,755 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,755 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,756 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,756 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,756 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,756 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,756 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4618c6346924:0, corePoolSize=2, maxPoolSize=2 2024-11-22T16:55:41,756 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,756 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,756 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,757 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,757 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,757 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4618c6346924:0, corePoolSize=1, maxPoolSize=1 2024-11-22T16:55:41,757 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:55:41,757 DEBUG [RS:0;4618c6346924:35523 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4618c6346924:0, corePoolSize=3, maxPoolSize=3 2024-11-22T16:55:41,758 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,758 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,758 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,758 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,758 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,759 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,35523,1732294539277-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:55:41,778 INFO [RS:0;4618c6346924:35523 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T16:55:41,780 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,35523,1732294539277-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,780 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,780 INFO [RS:0;4618c6346924:35523 {}] regionserver.Replication(171): 4618c6346924,35523,1732294539277 started 2024-11-22T16:55:41,799 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:55:41,799 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1482): Serving as 4618c6346924,35523,1732294539277, RpcServer on 4618c6346924/172.17.0.2:35523, sessionid=0x101409adc3d0001 2024-11-22T16:55:41,800 DEBUG [RS:0;4618c6346924:35523 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T16:55:41,800 DEBUG [RS:0;4618c6346924:35523 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4618c6346924,35523,1732294539277 2024-11-22T16:55:41,800 DEBUG [RS:0;4618c6346924:35523 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,35523,1732294539277' 2024-11-22T16:55:41,800 DEBUG [RS:0;4618c6346924:35523 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T16:55:41,802 DEBUG [RS:0;4618c6346924:35523 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T16:55:41,802 DEBUG [RS:0;4618c6346924:35523 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T16:55:41,802 DEBUG [RS:0;4618c6346924:35523 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T16:55:41,803 DEBUG [RS:0;4618c6346924:35523 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4618c6346924,35523,1732294539277 2024-11-22T16:55:41,803 DEBUG [RS:0;4618c6346924:35523 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4618c6346924,35523,1732294539277' 2024-11-22T16:55:41,803 DEBUG [RS:0;4618c6346924:35523 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T16:55:41,804 DEBUG [RS:0;4618c6346924:35523 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T16:55:41,804 DEBUG [RS:0;4618c6346924:35523 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T16:55:41,805 INFO [RS:0;4618c6346924:35523 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T16:55:41,805 INFO [RS:0;4618c6346924:35523 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T16:55:41,886 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:42,886 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:43,887 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:44,888 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:45,889 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:46,890 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:47,891 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:48,892 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:49,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-22T16:55:49,004 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-22T16:55:49,892 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:50,893 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:51,894 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:52,894 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:53,895 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:54,896 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:55,897 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:56,897 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:57,898 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:58,899 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:55:59,899 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:00,900 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:01,901 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:02,902 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:03,903 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:04,903 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:05,904 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:06,905 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:07,580 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T16:56:07,905 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:08,906 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:09,907 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:10,907 WARN [4618c6346924:38813 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T16:56:11,810 INFO [RS:0;4618c6346924:35523 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T16:56:11,815 INFO [RS:0;4618c6346924:35523 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4618c6346924%2C35523%2C1732294539277, suffix=, logDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/WALs/4618c6346924,35523,1732294539277, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/oldWALs, maxLogs=32 2024-11-22T16:56:11,833 DEBUG [RS:0;4618c6346924:35523 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/WALs/4618c6346924,35523,1732294539277/4618c6346924%2C35523%2C1732294539277.1732294571817, exclude list is [], retry=0 2024-11-22T16:56:11,839 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33941,DS-44b0140a-53fc-4308-ae89-5803ba993d1e,DISK] 2024-11-22T16:56:11,850 INFO [RS:0;4618c6346924:35523 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/WALs/4618c6346924,35523,1732294539277/4618c6346924%2C35523%2C1732294539277.1732294571817 2024-11-22T16:56:11,851 DEBUG [RS:0;4618c6346924:35523 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-22T16:56:11,911 DEBUG [4618c6346924:38813 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T16:56:11,928 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:11,939 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,35523,1732294539277, state=OPENING 2024-11-22T16:56:11,946 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T16:56:11,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:56:11,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:56:11,950 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:56:11,951 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:56:11,957 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T16:56:11,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:12,191 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T16:56:12,208 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35283, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T16:56:12,252 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T16:56:12,253 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T16:56:12,254 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T16:56:12,262 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4618c6346924%2C35523%2C1732294539277.meta, suffix=.meta, logDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/WALs/4618c6346924,35523,1732294539277, archiveDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/oldWALs, maxLogs=32 2024-11-22T16:56:12,297 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/WALs/4618c6346924,35523,1732294539277/4618c6346924%2C35523%2C1732294539277.meta.1732294572271.meta, exclude list is [], retry=0 2024-11-22T16:56:12,309 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33941,DS-44b0140a-53fc-4308-ae89-5803ba993d1e,DISK] 2024-11-22T16:56:12,323 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/WALs/4618c6346924,35523,1732294539277/4618c6346924%2C35523%2C1732294539277.meta.1732294572271.meta 2024-11-22T16:56:12,325 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45349:45349)] 2024-11-22T16:56:12,325 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:12,328 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T16:56:12,331 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T16:56:12,339 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T16:56:12,345 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T16:56:12,347 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:12,347 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T16:56:12,347 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T16:56:12,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-22T16:56:12,368 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T16:56:12,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:12,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:56:12,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-22T16:56:12,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T16:56:12,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:12,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:56:12,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-22T16:56:12,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T16:56:12,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:12,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:56:12,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-22T16:56:12,386 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T16:56:12,386 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:12,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T16:56:12,387 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T16:56:12,392 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740 2024-11-22T16:56:12,396 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740 2024-11-22T16:56:12,401 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T16:56:12,401 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T16:56:12,402 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T16:56:12,407 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T16:56:12,411 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69387476, jitterRate=0.033953964710235596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T16:56:12,412 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T16:56:12,416 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732294572348Writing region info on filesystem at 1732294572348Initializing all the Stores at 1732294572350 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732294572351 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732294572364 (+13 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294572364Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732294572364Cleaning up temporary data from old regions at 1732294572401 (+37 ms)Running coprocessor post-open hooks at 1732294572412 (+11 ms)Region opened successfully at 1732294572416 (+4 ms) 2024-11-22T16:56:12,426 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732294572154 2024-11-22T16:56:12,452 DEBUG [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T16:56:12,453 INFO [RS_OPEN_META-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T16:56:12,453 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:12,458 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4618c6346924,35523,1732294539277, state=OPEN 2024-11-22T16:56:12,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:56:12,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T16:56:12,466 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:56:12,466 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T16:56:12,467 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4618c6346924,35523,1732294539277 2024-11-22T16:56:12,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T16:56:12,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4618c6346924,35523,1732294539277 in 508 msec 2024-11-22T16:56:12,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T16:56:12,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.7580 sec 2024-11-22T16:56:12,489 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T16:56:12,489 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T16:56:12,520 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:56:12,522 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,35523,1732294539277, seqNum=-1] 2024-11-22T16:56:12,553 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:56:12,556 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35907, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:56:12,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0680 sec 2024-11-22T16:56:12,598 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732294572598, completionTime=-1 2024-11-22T16:56:12,605 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T16:56:12,605 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T16:56:12,645 INFO [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T16:56:12,646 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732294632645 2024-11-22T16:56:12,646 INFO [master/4618c6346924:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732294692646 2024-11-22T16:56:12,646 INFO [master/4618c6346924:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 40 msec 2024-11-22T16:56:12,648 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,38813,1732294538589-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T16:56:12,649 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,38813,1732294538589-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:56:12,649 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,38813,1732294538589-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:56:12,651 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4618c6346924:38813, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:56:12,651 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T16:56:12,652 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T16:56:12,660 DEBUG [master/4618c6346924:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T16:56:12,696 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.274sec 2024-11-22T16:56:12,701 INFO [master/4618c6346924:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T16:56:12,703 INFO [master/4618c6346924:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T16:56:12,704 INFO [master/4618c6346924:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T16:56:12,706 INFO [master/4618c6346924:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T16:56:12,706 INFO [master/4618c6346924:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T16:56:12,707 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,38813,1732294538589-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T16:56:12,708 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,38813,1732294538589-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T16:56:12,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f875e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:56:12,726 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T16:56:12,727 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T16:56:12,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4618c6346924,38813,-1 for getting cluster id 2024-11-22T16:56:12,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T16:56:12,739 DEBUG [master/4618c6346924:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T16:56:12,741 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T16:56:12,741 INFO [master/4618c6346924:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4618c6346924,38813,1732294538589-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T16:56:12,758 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5de1f6e5-c0b9-4345-b601-54e90cd293c4' 2024-11-22T16:56:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T16:56:12,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5de1f6e5-c0b9-4345-b601-54e90cd293c4" 2024-11-22T16:56:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f3a26d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:56:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4618c6346924,38813,-1] 2024-11-22T16:56:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T16:56:12,775 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T16:56:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:56:12,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ec3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:56:12,780 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:56:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,35523,1732294539277, seqNum=-1] 2024-11-22T16:56:12,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:56:12,795 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:56:12,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4618c6346924,38813,1732294538589 2024-11-22T16:56:12,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7c601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:56:12,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4618c6346924,38813,-1 for getting cluster id 2024-11-22T16:56:12,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T16:56:12,835 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5de1f6e5-c0b9-4345-b601-54e90cd293c4' 2024-11-22T16:56:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T16:56:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5de1f6e5-c0b9-4345-b601-54e90cd293c4" 2024-11-22T16:56:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44627131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:56:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4618c6346924,38813,-1] 2024-11-22T16:56:12,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T16:56:12,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:56:12,864 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T16:56:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9e7cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T16:56:12,897 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=294, ProcessCount=11, AvailableMemoryMB=7404 2024-11-22T16:56:12,909 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T16:56:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4618c6346924,38813,1732294538589 2024-11-22T16:56:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a1132fb 2024-11-22T16:56:12,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T16:56:12,939 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T16:56:12,956 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-22T16:56:12,983 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:12,989 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:12,993 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:12,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-22T16:56:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:56:13,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741835_1011 (size=428) 2024-11-22T16:56:13,146 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a7a1fb7610547cebdfcc43dd7732814, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:13,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741836_1012 (size=79) 2024-11-22T16:56:13,200 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:13,200 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 1a7a1fb7610547cebdfcc43dd7732814, disabling compactions & flushes 2024-11-22T16:56:13,200 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:56:13,200 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:56:13,200 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. after waiting 0 ms 2024-11-22T16:56:13,200 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:56:13,202 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:56:13,202 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a7a1fb7610547cebdfcc43dd7732814: Waiting for close lock at 1732294573200Disabling compacts and flushes for region at 1732294573200Disabling writes for close at 1732294573200Writing region close event to WAL at 1732294573202 (+2 ms)Closed at 1732294573202 2024-11-22T16:56:13,205 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:13,214 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732294573205"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294573205"}]},"ts":"1732294573205"} 2024-11-22T16:56:13,227 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:13,230 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:13,234 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294573230"}]},"ts":"1732294573230"} 2024-11-22T16:56:13,245 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-22T16:56:13,248 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1a7a1fb7610547cebdfcc43dd7732814, ASSIGN}] 2024-11-22T16:56:13,253 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1a7a1fb7610547cebdfcc43dd7732814, ASSIGN 2024-11-22T16:56:13,261 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1a7a1fb7610547cebdfcc43dd7732814, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:13,414 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a7a1fb7610547cebdfcc43dd7732814, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:13,421 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1a7a1fb7610547cebdfcc43dd7732814, ASSIGN because future has completed 2024-11-22T16:56:13,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a7a1fb7610547cebdfcc43dd7732814, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:13,613 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:56:13,614 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a7a1fb7610547cebdfcc43dd7732814, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:13,614 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,615 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:13,615 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,615 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,620 INFO [StoreOpener-1a7a1fb7610547cebdfcc43dd7732814-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,625 INFO [StoreOpener-1a7a1fb7610547cebdfcc43dd7732814-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a7a1fb7610547cebdfcc43dd7732814 columnFamilyName testFamily 2024-11-22T16:56:13,626 DEBUG [StoreOpener-1a7a1fb7610547cebdfcc43dd7732814-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:13,629 INFO [StoreOpener-1a7a1fb7610547cebdfcc43dd7732814-1 {}] regionserver.HStore(327): Store=1a7a1fb7610547cebdfcc43dd7732814/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:13,629 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,633 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,635 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,640 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,640 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,647 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,653 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:13,654 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1a7a1fb7610547cebdfcc43dd7732814; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68441261, jitterRate=0.019854262471199036}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:13,655 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:56:13,656 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1a7a1fb7610547cebdfcc43dd7732814: Running coprocessor pre-open hook at 1732294573615Writing region info on filesystem at 1732294573615Initializing all the Stores at 1732294573619 (+4 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294573619Cleaning up temporary data from old regions at 1732294573640 (+21 ms)Running coprocessor post-open hooks at 1732294573655 (+15 ms)Region opened successfully at 1732294573656 (+1 ms) 2024-11-22T16:56:13,660 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814., pid=6, masterSystemTime=1732294573599 2024-11-22T16:56:13,667 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:56:13,667 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:56:13,669 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a7a1fb7610547cebdfcc43dd7732814, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:13,677 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a7a1fb7610547cebdfcc43dd7732814, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:13,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T16:56:13,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a7a1fb7610547cebdfcc43dd7732814, server=4618c6346924,35523,1732294539277 in 247 msec 2024-11-22T16:56:13,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T16:56:13,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=1a7a1fb7610547cebdfcc43dd7732814, ASSIGN in 447 msec 2024-11-22T16:56:13,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:13,703 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294573703"}]},"ts":"1732294573703"} 2024-11-22T16:56:13,708 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-22T16:56:13,710 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:13,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 748 msec 2024-11-22T16:56:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T16:56:14,072 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-22T16:56:14,072 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-22T16:56:14,074 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:14,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-22T16:56:14,083 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:14,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-22T16:56:14,086 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T16:56:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4618c6346924,35523,1732294539277, seqNum=-1] 2024-11-22T16:56:14,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T16:56:14,110 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T16:56:14,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:14,140 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:14,144 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-22T16:56:14,152 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-22T16:56:14,158 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-22T16:56:14,161 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-22T16:56:14,161 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6968522250928521224 2024-11-22T16:56:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51164 deadline: 1732294577160, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-22T16:56:14,240 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-22T16:56:14,244 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-22T16:56:14,246 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:14,253 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:14,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6968522250928521226 2024-11-22T16:56:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51164 deadline: 1732294577257, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-22T16:56:14,265 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:14,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=6968522250928521227 2024-11-22T16:56:14,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51164 deadline: 1732294577264, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-22T16:56:14,271 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:14,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=6968522250928521228 2024-11-22T16:56:14,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51164 deadline: 1732294577273, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-22T16:56:14,280 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:14,300 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1915656722_22 at /127.0.0.1:59214 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4618c6346924:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=294 (was 294), ProcessCount=11 (was 11), AvailableMemoryMB=7350 (was 7404) 2024-11-22T16:56:14,345 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=294, ProcessCount=11, AvailableMemoryMB=7347 2024-11-22T16:56:14,351 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:14,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-22T16:56:14,357 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:14,357 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:14,358 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-22T16:56:14,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T16:56:14,361 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:14,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741837_1013 (size=414) 2024-11-22T16:56:14,839 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 62ab92dff0aec162c9c270ccc6cfe4c0, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:14,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741838_1014 (size=65) 2024-11-22T16:56:14,898 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:14,899 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 62ab92dff0aec162c9c270ccc6cfe4c0, disabling compactions & flushes 2024-11-22T16:56:14,899 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:56:14,899 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:56:14,899 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. after waiting 0 ms 2024-11-22T16:56:14,899 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:56:14,899 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:56:14,899 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 62ab92dff0aec162c9c270ccc6cfe4c0: Waiting for close lock at 1732294574898Disabling compacts and flushes for region at 1732294574898Disabling writes for close at 1732294574899 (+1 ms)Writing region close event to WAL at 1732294574899Closed at 1732294574899 2024-11-22T16:56:14,902 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:14,903 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732294574903"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294574903"}]},"ts":"1732294574903"} 2024-11-22T16:56:14,908 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:14,910 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:14,911 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294574910"}]},"ts":"1732294574910"} 2024-11-22T16:56:14,916 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-22T16:56:14,916 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=62ab92dff0aec162c9c270ccc6cfe4c0, ASSIGN}] 2024-11-22T16:56:14,920 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=62ab92dff0aec162c9c270ccc6cfe4c0, ASSIGN 2024-11-22T16:56:14,922 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=62ab92dff0aec162c9c270ccc6cfe4c0, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:15,075 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=62ab92dff0aec162c9c270ccc6cfe4c0, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:15,082 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=62ab92dff0aec162c9c270ccc6cfe4c0, ASSIGN because future has completed 2024-11-22T16:56:15,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62ab92dff0aec162c9c270ccc6cfe4c0, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:15,256 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:56:15,256 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 62ab92dff0aec162c9c270ccc6cfe4c0, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:15,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:15,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,260 INFO [StoreOpener-62ab92dff0aec162c9c270ccc6cfe4c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,262 INFO [StoreOpener-62ab92dff0aec162c9c270ccc6cfe4c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62ab92dff0aec162c9c270ccc6cfe4c0 columnFamilyName testFamily 2024-11-22T16:56:15,262 DEBUG [StoreOpener-62ab92dff0aec162c9c270ccc6cfe4c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:15,263 INFO [StoreOpener-62ab92dff0aec162c9c270ccc6cfe4c0-1 {}] regionserver.HStore(327): Store=62ab92dff0aec162c9c270ccc6cfe4c0/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:15,264 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,265 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,267 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,270 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,276 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:15,277 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 62ab92dff0aec162c9c270ccc6cfe4c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75414726, jitterRate=0.12376698851585388}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:15,277 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:56:15,279 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 62ab92dff0aec162c9c270ccc6cfe4c0: Running coprocessor pre-open hook at 1732294575257Writing region info on filesystem at 1732294575257Initializing all the Stores at 1732294575259 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294575259Cleaning up temporary data from old regions at 1732294575267 (+8 ms)Running coprocessor post-open hooks at 1732294575277 (+10 ms)Region opened successfully at 1732294575278 (+1 ms) 2024-11-22T16:56:15,280 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., pid=9, masterSystemTime=1732294575249 2024-11-22T16:56:15,285 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:56:15,285 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:56:15,286 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=62ab92dff0aec162c9c270ccc6cfe4c0, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:15,293 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 62ab92dff0aec162c9c270ccc6cfe4c0, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:15,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-22T16:56:15,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 62ab92dff0aec162c9c270ccc6cfe4c0, server=4618c6346924,35523,1732294539277 in 202 msec 2024-11-22T16:56:15,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-22T16:56:15,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=62ab92dff0aec162c9c270ccc6cfe4c0, ASSIGN in 384 msec 2024-11-22T16:56:15,308 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:15,308 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294575308"}]},"ts":"1732294575308"} 2024-11-22T16:56:15,311 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-22T16:56:15,313 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:15,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 962 msec 2024-11-22T16:56:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-22T16:56:15,371 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-22T16:56:15,371 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-22T16:56:15,371 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:15,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-22T16:56:15,377 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:15,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-22T16:56:15,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:15,401 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:15,401 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-22T16:56:15,407 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-22T16:56:15,409 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-22T16:56:17,887 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:56:17,919 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-22T16:56:17,921 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T16:56:17,923 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-22T16:56:19,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T16:56:19,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:19,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T16:56:19,004 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-22T16:56:19,005 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-22T16:56:19,005 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:19,006 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-22T16:56:19,006 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:21,419 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., hostname=4618c6346924,35523,1732294539277, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., hostname=4618c6346924,35523,1732294539277, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=4618c6346924:35523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-11-22T16:56:21,419 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., hostname=4618c6346924,35523,1732294539277, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6009ms, rpcTimeout=5999ms 2024-11-22T16:56:21,419 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., hostname=4618c6346924,35523,1732294539277, seqNum=2 from cache 2024-11-22T16:56:21,435 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=371 (was 294) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7292 (was 7347) 2024-11-22T16:56:21,445 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=371, ProcessCount=11, AvailableMemoryMB=7292 2024-11-22T16:56:21,449 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:21,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-22T16:56:21,453 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:21,453 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:21,454 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-22T16:56:21,455 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-22T16:56:21,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741839_1015 (size=412) 2024-11-22T16:56:21,474 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 23be4b4a4ddceb8e344defa4c78c05b3, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:21,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741840_1016 (size=63) 2024-11-22T16:56:21,492 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:21,492 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 23be4b4a4ddceb8e344defa4c78c05b3, disabling compactions & flushes 2024-11-22T16:56:21,493 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:56:21,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:56:21,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. after waiting 0 ms 2024-11-22T16:56:21,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:56:21,493 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:56:21,493 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 23be4b4a4ddceb8e344defa4c78c05b3: Waiting for close lock at 1732294581492Disabling compacts and flushes for region at 1732294581492Disabling writes for close at 1732294581493 (+1 ms)Writing region close event to WAL at 1732294581493Closed at 1732294581493 2024-11-22T16:56:21,495 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:21,495 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732294581495"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294581495"}]},"ts":"1732294581495"} 2024-11-22T16:56:21,499 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:21,501 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:21,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294581501"}]},"ts":"1732294581501"} 2024-11-22T16:56:21,504 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-22T16:56:21,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=23be4b4a4ddceb8e344defa4c78c05b3, ASSIGN}] 2024-11-22T16:56:21,506 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=23be4b4a4ddceb8e344defa4c78c05b3, ASSIGN 2024-11-22T16:56:21,508 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=23be4b4a4ddceb8e344defa4c78c05b3, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:21,659 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=23be4b4a4ddceb8e344defa4c78c05b3, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:21,663 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=23be4b4a4ddceb8e344defa4c78c05b3, ASSIGN because future has completed 2024-11-22T16:56:21,664 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23be4b4a4ddceb8e344defa4c78c05b3, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:21,825 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:56:21,825 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 23be4b4a4ddceb8e344defa4c78c05b3, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:21,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:21,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,830 INFO [StoreOpener-23be4b4a4ddceb8e344defa4c78c05b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,832 INFO [StoreOpener-23be4b4a4ddceb8e344defa4c78c05b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23be4b4a4ddceb8e344defa4c78c05b3 columnFamilyName testFamily 2024-11-22T16:56:21,833 DEBUG [StoreOpener-23be4b4a4ddceb8e344defa4c78c05b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:21,833 INFO [StoreOpener-23be4b4a4ddceb8e344defa4c78c05b3-1 {}] regionserver.HStore(327): Store=23be4b4a4ddceb8e344defa4c78c05b3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:21,834 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,836 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,836 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,841 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,844 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:21,846 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 23be4b4a4ddceb8e344defa4c78c05b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62103993, jitterRate=-0.07457838952541351}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:21,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:56:21,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 23be4b4a4ddceb8e344defa4c78c05b3: Running coprocessor pre-open hook at 1732294581826Writing region info on filesystem at 1732294581826Initializing all the Stores at 1732294581828 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294581828Cleaning up temporary data from old regions at 1732294581837 (+9 ms)Running coprocessor post-open hooks at 1732294581847 (+10 ms)Region opened successfully at 1732294581848 (+1 ms) 2024-11-22T16:56:21,850 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3., pid=12, masterSystemTime=1732294581818 2024-11-22T16:56:21,854 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:56:21,854 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:56:21,855 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=23be4b4a4ddceb8e344defa4c78c05b3, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:21,859 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23be4b4a4ddceb8e344defa4c78c05b3, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:21,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-22T16:56:21,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 23be4b4a4ddceb8e344defa4c78c05b3, server=4618c6346924,35523,1732294539277 in 198 msec 2024-11-22T16:56:21,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-22T16:56:21,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=23be4b4a4ddceb8e344defa4c78c05b3, ASSIGN in 363 msec 2024-11-22T16:56:21,873 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:21,874 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294581873"}]},"ts":"1732294581873"} 2024-11-22T16:56:21,878 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-22T16:56:21,880 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:21,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 431 msec 2024-11-22T16:56:21,911 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-22T16:56:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-22T16:56:22,471 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-22T16:56:22,471 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-22T16:56:22,472 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:22,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-22T16:56:22,478 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:22,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-22T16:56:22,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:22,498 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:22,498 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-22T16:56:22,504 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-22T16:56:22,506 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-22T16:56:22,508 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-22T16:56:22,509 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-22T16:56:23,924 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-22T16:56:27,328 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:56:28,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35523 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521237' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T16:56:28,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35523 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:51164 deadline: 1732294594515, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521237' does not exist 2024-11-22T16:56:28,520 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 4618c6346924,35523,1732294539277 for closing scanner id = 6968522250928521237 for 23be4b4a4ddceb8e344defa4c78c05b3 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521237' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:56:28,523 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 371), ProcessCount=11 (was 11), AvailableMemoryMB=7256 (was 7292) 2024-11-22T16:56:28,534 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=7256 2024-11-22T16:56:28,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-22T16:56:28,541 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:28,541 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:28,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-22T16:56:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T16:56:28,543 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:28,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741841_1017 (size=423) 2024-11-22T16:56:28,956 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 20cea65d6b97f32bfc869ed22ded698a, NAME => 'testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:28,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741842_1018 (size=74) 2024-11-22T16:56:29,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-22T16:56:29,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-22T16:56:29,373 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:29,374 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 20cea65d6b97f32bfc869ed22ded698a, disabling compactions & flushes 2024-11-22T16:56:29,374 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:56:29,374 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:56:29,374 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. after waiting 0 ms 2024-11-22T16:56:29,374 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:56:29,374 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:56:29,374 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 20cea65d6b97f32bfc869ed22ded698a: Waiting for close lock at 1732294589373Disabling compacts and flushes for region at 1732294589373Disabling writes for close at 1732294589374 (+1 ms)Writing region close event to WAL at 1732294589374Closed at 1732294589374 2024-11-22T16:56:29,376 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:29,376 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732294589376"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294589376"}]},"ts":"1732294589376"} 2024-11-22T16:56:29,381 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:29,383 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:29,383 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294589383"}]},"ts":"1732294589383"} 2024-11-22T16:56:29,386 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-22T16:56:29,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20cea65d6b97f32bfc869ed22ded698a, ASSIGN}] 2024-11-22T16:56:29,389 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20cea65d6b97f32bfc869ed22ded698a, ASSIGN 2024-11-22T16:56:29,391 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20cea65d6b97f32bfc869ed22ded698a, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:29,542 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=20cea65d6b97f32bfc869ed22ded698a, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:29,547 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20cea65d6b97f32bfc869ed22ded698a, ASSIGN because future has completed 2024-11-22T16:56:29,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20cea65d6b97f32bfc869ed22ded698a, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T16:56:29,710 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:56:29,710 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 20cea65d6b97f32bfc869ed22ded698a, NAME => 'testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:29,710 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,711 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:29,711 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,711 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,713 INFO [StoreOpener-20cea65d6b97f32bfc869ed22ded698a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,716 INFO [StoreOpener-20cea65d6b97f32bfc869ed22ded698a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20cea65d6b97f32bfc869ed22ded698a columnFamilyName testFamily 2024-11-22T16:56:29,717 DEBUG [StoreOpener-20cea65d6b97f32bfc869ed22ded698a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:29,718 INFO [StoreOpener-20cea65d6b97f32bfc869ed22ded698a-1 {}] regionserver.HStore(327): Store=20cea65d6b97f32bfc869ed22ded698a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:29,718 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,719 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,719 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,720 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,720 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,723 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,727 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:29,727 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 20cea65d6b97f32bfc869ed22ded698a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64039093, jitterRate=-0.045743152499198914}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:29,728 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:56:29,728 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 20cea65d6b97f32bfc869ed22ded698a: Running coprocessor pre-open hook at 1732294589711Writing region info on filesystem at 1732294589711Initializing all the Stores at 1732294589713 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294589713Cleaning up temporary data from old regions at 1732294589720 (+7 ms)Running coprocessor post-open hooks at 1732294589728 (+8 ms)Region opened successfully at 1732294589728 2024-11-22T16:56:29,730 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a., pid=15, masterSystemTime=1732294589703 2024-11-22T16:56:29,735 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=20cea65d6b97f32bfc869ed22ded698a, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:29,736 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:56:29,736 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:56:29,739 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20cea65d6b97f32bfc869ed22ded698a, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:29,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-22T16:56:29,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 20cea65d6b97f32bfc869ed22ded698a, server=4618c6346924,35523,1732294539277 in 193 msec 2024-11-22T16:56:29,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-22T16:56:29,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=20cea65d6b97f32bfc869ed22ded698a, ASSIGN in 358 msec 2024-11-22T16:56:29,753 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:29,753 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294589753"}]},"ts":"1732294589753"} 2024-11-22T16:56:29,756 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-22T16:56:29,758 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:29,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2210 sec 2024-11-22T16:56:29,925 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-22T16:56:31,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-22T16:56:31,561 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-22T16:56:31,561 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-22T16:56:31,561 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:31,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-22T16:56:31,567 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:31,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-22T16:56:31,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:31,583 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:31,583 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-22T16:56:31,586 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-22T16:56:31,588 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-22T16:56:31,589 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-22T16:56:31,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6968522250928521241 2024-11-22T16:56:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51164 deadline: 1732294594589, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-22T16:56:31,594 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-22T16:56:31,595 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-22T16:56:31,596 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:31,597 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:31,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=6968522250928521243 2024-11-22T16:56:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51164 deadline: 1732294594597, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-22T16:56:31,601 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:31,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=6968522250928521244 2024-11-22T16:56:31,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51164 deadline: 1732294594601, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-22T16:56:31,606 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:31,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=6968522250928521245 2024-11-22T16:56:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.2:51164 deadline: 1732294594605, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-22T16:56:31,610 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-22T16:56:31,621 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1915656722_22 at /127.0.0.1:44766 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=376 (was 366) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7245 (was 7256) 2024-11-22T16:56:31,639 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=376, ProcessCount=11, AvailableMemoryMB=7245 2024-11-22T16:56:31,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-22T16:56:31,646 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:31,646 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:31,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-22T16:56:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-22T16:56:31,648 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:31,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741843_1019 (size=418) 2024-11-22T16:56:31,662 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5910e30db9d5827c0e378da14e38ccfa, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:31,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741844_1020 (size=69) 2024-11-22T16:56:31,677 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:31,677 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 5910e30db9d5827c0e378da14e38ccfa, disabling compactions & flushes 2024-11-22T16:56:31,677 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:56:31,677 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:56:31,677 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. after waiting 0 ms 2024-11-22T16:56:31,677 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:56:31,677 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:56:31,678 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5910e30db9d5827c0e378da14e38ccfa: Waiting for close lock at 1732294591677Disabling compacts and flushes for region at 1732294591677Disabling writes for close at 1732294591677Writing region close event to WAL at 1732294591677Closed at 1732294591677 2024-11-22T16:56:31,685 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:31,685 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732294591685"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294591685"}]},"ts":"1732294591685"} 2024-11-22T16:56:31,689 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:31,691 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:31,692 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294591692"}]},"ts":"1732294591692"} 2024-11-22T16:56:31,695 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-22T16:56:31,695 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5910e30db9d5827c0e378da14e38ccfa, ASSIGN}] 2024-11-22T16:56:31,697 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5910e30db9d5827c0e378da14e38ccfa, ASSIGN 2024-11-22T16:56:31,698 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5910e30db9d5827c0e378da14e38ccfa, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:31,850 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5910e30db9d5827c0e378da14e38ccfa, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:31,854 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5910e30db9d5827c0e378da14e38ccfa, ASSIGN because future has completed 2024-11-22T16:56:31,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5910e30db9d5827c0e378da14e38ccfa, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:32,014 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:56:32,014 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 5910e30db9d5827c0e378da14e38ccfa, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:32,014 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,014 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:32,015 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,015 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,017 INFO [StoreOpener-5910e30db9d5827c0e378da14e38ccfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,019 INFO [StoreOpener-5910e30db9d5827c0e378da14e38ccfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5910e30db9d5827c0e378da14e38ccfa columnFamilyName testFamily 2024-11-22T16:56:32,019 DEBUG [StoreOpener-5910e30db9d5827c0e378da14e38ccfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:32,020 INFO [StoreOpener-5910e30db9d5827c0e378da14e38ccfa-1 {}] regionserver.HStore(327): Store=5910e30db9d5827c0e378da14e38ccfa/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:32,020 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,021 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,022 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,022 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,022 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,024 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,027 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:32,028 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 5910e30db9d5827c0e378da14e38ccfa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73844946, jitterRate=0.10037544369697571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:32,028 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:56:32,029 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 5910e30db9d5827c0e378da14e38ccfa: Running coprocessor pre-open hook at 1732294592015Writing region info on filesystem at 1732294592015Initializing all the Stores at 1732294592016 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294592016Cleaning up temporary data from old regions at 1732294592022 (+6 ms)Running coprocessor post-open hooks at 1732294592028 (+6 ms)Region opened successfully at 1732294592029 (+1 ms) 2024-11-22T16:56:32,030 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., pid=18, masterSystemTime=1732294592008 2024-11-22T16:56:32,033 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:56:32,033 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:56:32,033 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5910e30db9d5827c0e378da14e38ccfa, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:32,036 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5910e30db9d5827c0e378da14e38ccfa, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:32,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-22T16:56:32,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 5910e30db9d5827c0e378da14e38ccfa, server=4618c6346924,35523,1732294539277 in 183 msec 2024-11-22T16:56:32,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-22T16:56:32,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5910e30db9d5827c0e378da14e38ccfa, ASSIGN in 346 msec 2024-11-22T16:56:32,044 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:32,045 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294592044"}]},"ts":"1732294592044"} 2024-11-22T16:56:32,047 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-22T16:56:32,048 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:32,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 406 msec 2024-11-22T16:56:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-22T16:56:32,661 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-22T16:56:32,661 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-22T16:56:32,661 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:32,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-22T16:56:32,666 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:32,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-22T16:56:32,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:32,679 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:32,680 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-22T16:56:32,682 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-22T16:56:32,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-22T16:56:33,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=4618c6346924:35523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-22T16:56:33,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-11-22T16:56:33,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2 from cache 2024-11-22T16:56:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-22T16:56:34,702 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:34,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-22T16:56:35,212 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:56:35,691 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=4618c6346924:35523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=988ms, rpcTimeout=980ms 2024-11-22T16:56:35,692 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=988ms, rpcTimeout=980ms 2024-11-22T16:56:35,692 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., hostname=4618c6346924,35523,1732294539277, seqNum=2 from cache 2024-11-22T16:56:35,701 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228 (was 228), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=346 (was 376), ProcessCount=11 (was 11), AvailableMemoryMB=7252 (was 7245) - AvailableMemoryMB LEAK? - 2024-11-22T16:56:35,709 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=228, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=346, ProcessCount=11, AvailableMemoryMB=7251 2024-11-22T16:56:35,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-22T16:56:35,716 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:35,716 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:35,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-22T16:56:35,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T16:56:35,717 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:35,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741845_1021 (size=416) 2024-11-22T16:56:35,927 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-22T16:56:36,128 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d84c6aa3b992bfbb823675e8d1ba55f, NAME => 'testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:36,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741846_1022 (size=67) 2024-11-22T16:56:36,205 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-22T16:56:36,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:36,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 2d84c6aa3b992bfbb823675e8d1ba55f, disabling compactions & flushes 2024-11-22T16:56:36,538 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:56:36,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:56:36,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. after waiting 0 ms 2024-11-22T16:56:36,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:56:36,538 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:56:36,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d84c6aa3b992bfbb823675e8d1ba55f: Waiting for close lock at 1732294596538Disabling compacts and flushes for region at 1732294596538Disabling writes for close at 1732294596538Writing region close event to WAL at 1732294596538Closed at 1732294596538 2024-11-22T16:56:36,540 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:36,541 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732294596540"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294596540"}]},"ts":"1732294596540"} 2024-11-22T16:56:36,543 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:36,544 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:36,544 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294596544"}]},"ts":"1732294596544"} 2024-11-22T16:56:36,547 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-22T16:56:36,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2d84c6aa3b992bfbb823675e8d1ba55f, ASSIGN}] 2024-11-22T16:56:36,548 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2d84c6aa3b992bfbb823675e8d1ba55f, ASSIGN 2024-11-22T16:56:36,549 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2d84c6aa3b992bfbb823675e8d1ba55f, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:36,700 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=2d84c6aa3b992bfbb823675e8d1ba55f, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:36,703 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2d84c6aa3b992bfbb823675e8d1ba55f, ASSIGN because future has completed 2024-11-22T16:56:36,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d84c6aa3b992bfbb823675e8d1ba55f, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T16:56:36,862 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:56:36,862 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d84c6aa3b992bfbb823675e8d1ba55f, NAME => 'testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:36,863 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,863 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:36,863 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,863 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,865 INFO [StoreOpener-2d84c6aa3b992bfbb823675e8d1ba55f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,867 INFO [StoreOpener-2d84c6aa3b992bfbb823675e8d1ba55f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d84c6aa3b992bfbb823675e8d1ba55f columnFamilyName testFamily 2024-11-22T16:56:36,867 DEBUG [StoreOpener-2d84c6aa3b992bfbb823675e8d1ba55f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:36,868 INFO [StoreOpener-2d84c6aa3b992bfbb823675e8d1ba55f-1 {}] regionserver.HStore(327): Store=2d84c6aa3b992bfbb823675e8d1ba55f/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:36,868 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,869 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,870 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,870 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,870 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,873 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,876 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:36,876 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 2d84c6aa3b992bfbb823675e8d1ba55f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65632634, jitterRate=-0.021997541189193726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:36,876 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:56:36,877 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 2d84c6aa3b992bfbb823675e8d1ba55f: Running coprocessor pre-open hook at 1732294596864Writing region info on filesystem at 1732294596864Initializing all the Stores at 1732294596865 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294596865Cleaning up temporary data from old regions at 1732294596870 (+5 ms)Running coprocessor post-open hooks at 1732294596876 (+6 ms)Region opened successfully at 1732294596877 (+1 ms) 2024-11-22T16:56:36,878 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f., pid=21, masterSystemTime=1732294596856 2024-11-22T16:56:36,881 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:56:36,881 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:56:36,882 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=2d84c6aa3b992bfbb823675e8d1ba55f, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:36,884 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d84c6aa3b992bfbb823675e8d1ba55f, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:36,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-22T16:56:36,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d84c6aa3b992bfbb823675e8d1ba55f, server=4618c6346924,35523,1732294539277 in 181 msec 2024-11-22T16:56:36,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-22T16:56:36,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=2d84c6aa3b992bfbb823675e8d1ba55f, ASSIGN in 341 msec 2024-11-22T16:56:36,892 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:36,892 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294596892"}]},"ts":"1732294596892"} 2024-11-22T16:56:36,894 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-22T16:56:36,895 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:36,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.1830 sec 2024-11-22T16:56:37,581 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T16:56:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-22T16:56:38,760 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-22T16:56:38,760 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-22T16:56:38,760 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:38,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-22T16:56:38,766 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:38,766 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-22T16:56:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:38,780 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:38,780 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-22T16:56:38,783 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-22T16:56:38,785 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-22T16:56:38,786 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-22T16:56:38,787 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-22T16:56:39,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-22T16:56:39,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:39,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-22T16:56:39,003 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:39,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-22T16:56:39,004 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:41,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35523 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521256' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T16:56:41,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35523 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:51164 deadline: 1732294602792, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521256' does not exist 2024-11-22T16:56:41,795 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 4618c6346924,35523,1732294539277 for closing scanner id = 6968522250928521256 for 2d84c6aa3b992bfbb823675e8d1ba55f of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521256' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:56:41,802 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 228), OpenFileDescriptor=448 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=318 (was 346), ProcessCount=11 (was 11), AvailableMemoryMB=7241 (was 7251) 2024-11-22T16:56:41,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=318, ProcessCount=11, AvailableMemoryMB=7241 2024-11-22T16:56:41,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-22T16:56:41,820 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:41,821 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:41,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-22T16:56:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-22T16:56:41,822 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:41,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741847_1023 (size=411) 2024-11-22T16:56:42,236 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3edce1f8cf628b4ac8b7aa6984f09414, NAME => 'testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:42,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741848_1024 (size=62) 2024-11-22T16:56:42,253 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:42,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 3edce1f8cf628b4ac8b7aa6984f09414, disabling compactions & flushes 2024-11-22T16:56:42,254 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:56:42,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:56:42,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. after waiting 0 ms 2024-11-22T16:56:42,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:56:42,254 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:56:42,254 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3edce1f8cf628b4ac8b7aa6984f09414: Waiting for close lock at 1732294602254Disabling compacts and flushes for region at 1732294602254Disabling writes for close at 1732294602254Writing region close event to WAL at 1732294602254Closed at 1732294602254 2024-11-22T16:56:42,256 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:42,257 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732294602256"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294602256"}]},"ts":"1732294602256"} 2024-11-22T16:56:42,260 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:42,262 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:42,262 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294602262"}]},"ts":"1732294602262"} 2024-11-22T16:56:42,265 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-22T16:56:42,265 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=3edce1f8cf628b4ac8b7aa6984f09414, ASSIGN}] 2024-11-22T16:56:42,267 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=3edce1f8cf628b4ac8b7aa6984f09414, ASSIGN 2024-11-22T16:56:42,269 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=3edce1f8cf628b4ac8b7aa6984f09414, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-22T16:56:42,365 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:56:42,396 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-22T16:56:42,421 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=3edce1f8cf628b4ac8b7aa6984f09414, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:42,424 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=3edce1f8cf628b4ac8b7aa6984f09414, ASSIGN because future has completed 2024-11-22T16:56:42,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3edce1f8cf628b4ac8b7aa6984f09414, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:42,588 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:56:42,588 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 3edce1f8cf628b4ac8b7aa6984f09414, NAME => 'testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:42,589 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,589 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:42,589 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,589 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,594 INFO [StoreOpener-3edce1f8cf628b4ac8b7aa6984f09414-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,597 INFO [StoreOpener-3edce1f8cf628b4ac8b7aa6984f09414-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3edce1f8cf628b4ac8b7aa6984f09414 columnFamilyName testFamily 2024-11-22T16:56:42,597 DEBUG [StoreOpener-3edce1f8cf628b4ac8b7aa6984f09414-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:42,598 INFO [StoreOpener-3edce1f8cf628b4ac8b7aa6984f09414-1 {}] regionserver.HStore(327): Store=3edce1f8cf628b4ac8b7aa6984f09414/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:42,598 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,600 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,600 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,601 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,601 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,605 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,609 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:42,611 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 3edce1f8cf628b4ac8b7aa6984f09414; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61193957, jitterRate=-0.08813898265361786}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:42,611 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:56:42,612 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 3edce1f8cf628b4ac8b7aa6984f09414: Running coprocessor pre-open hook at 1732294602589Writing region info on filesystem at 1732294602589Initializing all the Stores at 1732294602593 (+4 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294602593Cleaning up temporary data from old regions at 1732294602601 (+8 ms)Running coprocessor post-open hooks at 1732294602611 (+10 ms)Region opened successfully at 1732294602611 2024-11-22T16:56:42,615 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414., pid=24, masterSystemTime=1732294602578 2024-11-22T16:56:42,618 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:56:42,618 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:56:42,621 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=3edce1f8cf628b4ac8b7aa6984f09414, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:42,625 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3edce1f8cf628b4ac8b7aa6984f09414, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:42,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-22T16:56:42,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 3edce1f8cf628b4ac8b7aa6984f09414, server=4618c6346924,35523,1732294539277 in 202 msec 2024-11-22T16:56:42,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-22T16:56:42,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=3edce1f8cf628b4ac8b7aa6984f09414, ASSIGN in 366 msec 2024-11-22T16:56:42,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:42,637 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294602637"}]},"ts":"1732294602637"} 2024-11-22T16:56:42,641 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-22T16:56:42,642 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:42,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 825 msec 2024-11-22T16:56:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-22T16:56:42,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-22T16:56:42,831 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-22T16:56:42,831 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:42,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-22T16:56:42,837 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:42,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-22T16:56:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:42,853 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:42,853 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-22T16:56:42,857 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-22T16:56:42,859 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-22T16:56:42,860 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-22T16:56:42,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-22T16:56:45,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35523 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521260' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T16:56:45,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35523 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:51164 deadline: 1732294606862, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521260' does not exist 2024-11-22T16:56:45,866 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 4618c6346924,35523,1732294539277 for closing scanner id = 6968522250928521260 for 3edce1f8cf628b4ac8b7aa6984f09414 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521260' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:56:45,873 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1915656722_22 at /127.0.0.1:52674 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=341 (was 318) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7205 (was 7241) 2024-11-22T16:56:45,882 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=341, ProcessCount=11, AvailableMemoryMB=7205 2024-11-22T16:56:45,885 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:45,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-22T16:56:45,888 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:45,889 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:45,889 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-22T16:56:45,890 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T16:56:45,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741849_1025 (size=419) 2024-11-22T16:56:45,908 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 06d748ca8bbb56ba14ac063439728986, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:45,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741850_1026 (size=70) 2024-11-22T16:56:45,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:45,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 06d748ca8bbb56ba14ac063439728986, disabling compactions & flushes 2024-11-22T16:56:45,917 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:56:45,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:56:45,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. after waiting 0 ms 2024-11-22T16:56:45,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:56:45,917 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:56:45,917 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 06d748ca8bbb56ba14ac063439728986: Waiting for close lock at 1732294605917Disabling compacts and flushes for region at 1732294605917Disabling writes for close at 1732294605917Writing region close event to WAL at 1732294605917Closed at 1732294605917 2024-11-22T16:56:45,919 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:45,919 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732294605919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294605919"}]},"ts":"1732294605919"} 2024-11-22T16:56:45,923 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:45,924 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:45,924 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294605924"}]},"ts":"1732294605924"} 2024-11-22T16:56:45,927 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-22T16:56:45,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=06d748ca8bbb56ba14ac063439728986, ASSIGN}] 2024-11-22T16:56:45,929 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=06d748ca8bbb56ba14ac063439728986, ASSIGN 2024-11-22T16:56:45,930 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=06d748ca8bbb56ba14ac063439728986, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:46,081 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=06d748ca8bbb56ba14ac063439728986, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:46,085 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=06d748ca8bbb56ba14ac063439728986, ASSIGN because future has completed 2024-11-22T16:56:46,085 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d748ca8bbb56ba14ac063439728986, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:46,245 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:56:46,245 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 06d748ca8bbb56ba14ac063439728986, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:46,246 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,246 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:46,246 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,246 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,248 INFO [StoreOpener-06d748ca8bbb56ba14ac063439728986-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,250 INFO [StoreOpener-06d748ca8bbb56ba14ac063439728986-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06d748ca8bbb56ba14ac063439728986 columnFamilyName testFamily 2024-11-22T16:56:46,251 DEBUG [StoreOpener-06d748ca8bbb56ba14ac063439728986-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:46,251 INFO [StoreOpener-06d748ca8bbb56ba14ac063439728986-1 {}] regionserver.HStore(327): Store=06d748ca8bbb56ba14ac063439728986/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:46,251 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,254 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,254 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,260 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:46,261 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 06d748ca8bbb56ba14ac063439728986; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65909701, jitterRate=-0.017868921160697937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:46,261 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:56:46,262 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 06d748ca8bbb56ba14ac063439728986: Running coprocessor pre-open hook at 1732294606246Writing region info on filesystem at 1732294606246Initializing all the Stores at 1732294606248 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294606248Cleaning up temporary data from old regions at 1732294606254 (+6 ms)Running coprocessor post-open hooks at 1732294606261 (+7 ms)Region opened successfully at 1732294606262 (+1 ms) 2024-11-22T16:56:46,263 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986., pid=27, masterSystemTime=1732294606239 2024-11-22T16:56:46,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:56:46,267 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:56:46,267 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=06d748ca8bbb56ba14ac063439728986, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:46,271 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06d748ca8bbb56ba14ac063439728986, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:46,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-22T16:56:46,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 06d748ca8bbb56ba14ac063439728986, server=4618c6346924,35523,1732294539277 in 187 msec 2024-11-22T16:56:46,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-22T16:56:46,278 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=06d748ca8bbb56ba14ac063439728986, ASSIGN in 348 msec 2024-11-22T16:56:46,280 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:46,280 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294606280"}]},"ts":"1732294606280"} 2024-11-22T16:56:46,283 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-22T16:56:46,284 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:46,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 399 msec 2024-11-22T16:56:46,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-22T16:56:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-22T16:56:46,911 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-22T16:56:46,911 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-22T16:56:46,911 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:46,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-22T16:56:46,917 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:46,917 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-22T16:56:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:46,933 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:46,933 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-22T16:56:46,936 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-22T16:56:46,938 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-22T16:56:48,091 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:56:48,397 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-22T16:56:48,398 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-22T16:56:49,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-22T16:56:49,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:49,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-22T16:56:49,003 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-22T16:56:52,941 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986., hostname=4618c6346924,35523,1732294539277, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986., hostname=4618c6346924,35523,1732294539277, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=4618c6346924:35523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-22T16:56:52,942 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986., hostname=4618c6346924,35523,1732294539277, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6004ms, rpcTimeout=5999ms 2024-11-22T16:56:52,942 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986., hostname=4618c6346924,35523,1732294539277, seqNum=2 from cache 2024-11-22T16:56:52,954 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=362 (was 341) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7191 (was 7205) 2024-11-22T16:56:52,963 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=362, ProcessCount=11, AvailableMemoryMB=7191 2024-11-22T16:56:52,966 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:56:52,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-22T16:56:52,970 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:56:52,970 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:52,970 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-22T16:56:52,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-22T16:56:52,972 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:56:52,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741851_1027 (size=407) 2024-11-22T16:56:52,991 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3d4d5f2feb4a5ec59970ac2ede0a2ce, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:56:53,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741852_1028 (size=58) 2024-11-22T16:56:53,417 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:53,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing d3d4d5f2feb4a5ec59970ac2ede0a2ce, disabling compactions & flushes 2024-11-22T16:56:53,418 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:56:53,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:56:53,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. after waiting 0 ms 2024-11-22T16:56:53,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:56:53,418 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:56:53,418 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3d4d5f2feb4a5ec59970ac2ede0a2ce: Waiting for close lock at 1732294613417Disabling compacts and flushes for region at 1732294613417Disabling writes for close at 1732294613418 (+1 ms)Writing region close event to WAL at 1732294613418Closed at 1732294613418 2024-11-22T16:56:53,420 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:56:53,420 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732294613420"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294613420"}]},"ts":"1732294613420"} 2024-11-22T16:56:53,423 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:56:53,425 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:56:53,425 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294613425"}]},"ts":"1732294613425"} 2024-11-22T16:56:53,428 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-22T16:56:53,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d3d4d5f2feb4a5ec59970ac2ede0a2ce, ASSIGN}] 2024-11-22T16:56:53,430 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d3d4d5f2feb4a5ec59970ac2ede0a2ce, ASSIGN 2024-11-22T16:56:53,431 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d3d4d5f2feb4a5ec59970ac2ede0a2ce, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:56:53,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-22T16:56:53,574 INFO [master/4618c6346924:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-22T16:56:53,574 INFO [master/4618c6346924:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-22T16:56:53,582 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d3d4d5f2feb4a5ec59970ac2ede0a2ce, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:53,586 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d3d4d5f2feb4a5ec59970ac2ede0a2ce, ASSIGN because future has completed 2024-11-22T16:56:53,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3d4d5f2feb4a5ec59970ac2ede0a2ce, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:56:53,750 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:56:53,750 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => d3d4d5f2feb4a5ec59970ac2ede0a2ce, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:56:53,750 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:56:53,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,753 INFO [StoreOpener-d3d4d5f2feb4a5ec59970ac2ede0a2ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,755 INFO [StoreOpener-d3d4d5f2feb4a5ec59970ac2ede0a2ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3d4d5f2feb4a5ec59970ac2ede0a2ce columnFamilyName testFamily 2024-11-22T16:56:53,755 DEBUG [StoreOpener-d3d4d5f2feb4a5ec59970ac2ede0a2ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:56:53,756 INFO [StoreOpener-d3d4d5f2feb4a5ec59970ac2ede0a2ce-1 {}] regionserver.HStore(327): Store=d3d4d5f2feb4a5ec59970ac2ede0a2ce/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:56:53,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,757 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,758 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,759 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,761 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,766 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:56:53,766 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened d3d4d5f2feb4a5ec59970ac2ede0a2ce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68077007, jitterRate=0.014426454901695251}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:56:53,767 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:56:53,768 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for d3d4d5f2feb4a5ec59970ac2ede0a2ce: Running coprocessor pre-open hook at 1732294613751Writing region info on filesystem at 1732294613751Initializing all the Stores at 1732294613752 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294613753 (+1 ms)Cleaning up temporary data from old regions at 1732294613759 (+6 ms)Running coprocessor post-open hooks at 1732294613767 (+8 ms)Region opened successfully at 1732294613768 (+1 ms) 2024-11-22T16:56:53,770 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce., pid=30, masterSystemTime=1732294613740 2024-11-22T16:56:53,773 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:56:53,773 INFO [RS_OPEN_PRIORITY_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:56:53,775 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=d3d4d5f2feb4a5ec59970ac2ede0a2ce, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:56:53,778 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3d4d5f2feb4a5ec59970ac2ede0a2ce, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:56:53,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-22T16:56:53,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure d3d4d5f2feb4a5ec59970ac2ede0a2ce, server=4618c6346924,35523,1732294539277 in 193 msec 2024-11-22T16:56:53,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-22T16:56:53,786 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=d3d4d5f2feb4a5ec59970ac2ede0a2ce, ASSIGN in 354 msec 2024-11-22T16:56:53,786 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:56:53,787 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294613787"}]},"ts":"1732294613787"} 2024-11-22T16:56:53,789 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-22T16:56:53,790 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:56:53,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 823 msec 2024-11-22T16:56:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-22T16:56:53,980 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-22T16:56:53,980 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-22T16:56:53,981 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:53,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-22T16:56:53,986 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:56:53,986 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-22T16:56:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:56:54,000 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:56:54,000 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-22T16:56:54,003 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-22T16:56:54,006 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-22T16:56:54,007 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-22T16:56:54,008 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-22T16:56:54,399 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-22T16:56:59,289 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:56:59,318 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-22T16:56:59,318 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-22T16:57:00,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35523 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521268' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-22T16:57:00,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35523 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.2:51164 deadline: 1732294626012, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521268' does not exist 2024-11-22T16:57:00,015 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 4618c6346924,35523,1732294539277 for closing scanner id = 6968522250928521268 for d3d4d5f2feb4a5ec59970ac2ede0a2ce of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '6968522250928521268' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T16:57:00,020 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 362) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7182 (was 7191) 2024-11-22T16:57:00,028 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=7181 2024-11-22T16:57:00,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T16:57:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-22T16:57:00,033 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T16:57:00,033 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:57:00,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-22T16:57:00,034 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T16:57:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T16:57:00,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741853_1029 (size=423) 2024-11-22T16:57:00,444 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 744b5f7fd7cfd3ba3e97d73d358d5754, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d 2024-11-22T16:57:00,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741854_1030 (size=74) 2024-11-22T16:57:00,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-22T16:57:00,853 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:57:00,853 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 744b5f7fd7cfd3ba3e97d73d358d5754, disabling compactions & flushes 2024-11-22T16:57:00,853 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:00,853 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:00,853 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. after waiting 0 ms 2024-11-22T16:57:00,853 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:00,853 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:00,853 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 744b5f7fd7cfd3ba3e97d73d358d5754: Waiting for close lock at 1732294620853Disabling compacts and flushes for region at 1732294620853Disabling writes for close at 1732294620853Writing region close event to WAL at 1732294620853Closed at 1732294620853 2024-11-22T16:57:00,855 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T16:57:00,855 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732294620855"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732294620855"}]},"ts":"1732294620855"} 2024-11-22T16:57:00,858 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T16:57:00,859 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T16:57:00,859 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294620859"}]},"ts":"1732294620859"} 2024-11-22T16:57:00,861 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-22T16:57:00,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=744b5f7fd7cfd3ba3e97d73d358d5754, ASSIGN}] 2024-11-22T16:57:00,863 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=744b5f7fd7cfd3ba3e97d73d358d5754, ASSIGN 2024-11-22T16:57:00,864 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=744b5f7fd7cfd3ba3e97d73d358d5754, ASSIGN; state=OFFLINE, location=4618c6346924,35523,1732294539277; forceNewPlan=false, retain=false 2024-11-22T16:57:01,014 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=744b5f7fd7cfd3ba3e97d73d358d5754, regionState=OPENING, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:57:01,018 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=744b5f7fd7cfd3ba3e97d73d358d5754, ASSIGN because future has completed 2024-11-22T16:57:01,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 744b5f7fd7cfd3ba3e97d73d358d5754, server=4618c6346924,35523,1732294539277}] 2024-11-22T16:57:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T16:57:01,175 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:01,176 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 744b5f7fd7cfd3ba3e97d73d358d5754, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754.', STARTKEY => '', ENDKEY => ''} 2024-11-22T16:57:01,176 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,176 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T16:57:01,176 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,176 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,178 INFO [StoreOpener-744b5f7fd7cfd3ba3e97d73d358d5754-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,180 INFO [StoreOpener-744b5f7fd7cfd3ba3e97d73d358d5754-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 744b5f7fd7cfd3ba3e97d73d358d5754 columnFamilyName testFamily 2024-11-22T16:57:01,180 DEBUG [StoreOpener-744b5f7fd7cfd3ba3e97d73d358d5754-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T16:57:01,180 INFO [StoreOpener-744b5f7fd7cfd3ba3e97d73d358d5754-1 {}] regionserver.HStore(327): Store=744b5f7fd7cfd3ba3e97d73d358d5754/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T16:57:01,181 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,182 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,182 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,183 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,183 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,184 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,187 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T16:57:01,188 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 744b5f7fd7cfd3ba3e97d73d358d5754; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60978453, jitterRate=-0.0913502424955368}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T16:57:01,188 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:01,189 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 744b5f7fd7cfd3ba3e97d73d358d5754: Running coprocessor pre-open hook at 1732294621176Writing region info on filesystem at 1732294621176Initializing all the Stores at 1732294621178 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732294621178Cleaning up temporary data from old regions at 1732294621183 (+5 ms)Running coprocessor post-open hooks at 1732294621188 (+5 ms)Region opened successfully at 1732294621189 (+1 ms) 2024-11-22T16:57:01,190 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., pid=33, masterSystemTime=1732294621170 2024-11-22T16:57:01,193 DEBUG [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:01,193 INFO [RS_OPEN_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:01,193 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=744b5f7fd7cfd3ba3e97d73d358d5754, regionState=OPEN, openSeqNum=2, regionLocation=4618c6346924,35523,1732294539277 2024-11-22T16:57:01,196 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 744b5f7fd7cfd3ba3e97d73d358d5754, server=4618c6346924,35523,1732294539277 because future has completed 2024-11-22T16:57:01,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-22T16:57:01,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 744b5f7fd7cfd3ba3e97d73d358d5754, server=4618c6346924,35523,1732294539277 in 179 msec 2024-11-22T16:57:01,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-22T16:57:01,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=744b5f7fd7cfd3ba3e97d73d358d5754, ASSIGN in 338 msec 2024-11-22T16:57:01,203 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T16:57:01,203 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732294621203"}]},"ts":"1732294621203"} 2024-11-22T16:57:01,205 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-22T16:57:01,206 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T16:57:01,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1750 sec 2024-11-22T16:57:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-22T16:57:03,061 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-22T16:57:03,061 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-22T16:57:03,061 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:57:03,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-22T16:57:03,066 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T16:57:03,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-22T16:57:03,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:57:03,079 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-22T16:57:03,079 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-22T16:57:03,082 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-22T16:57:03,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-22T16:57:04,092 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=4618c6346924:35523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-22T16:57:04,092 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-11-22T16:57:04,092 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2 from cache 2024-11-22T16:57:04,584 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-22T16:57:05,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2] 2024-11-22T16:57:05,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35523 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-22T16:57:06,091 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=4618c6346924:35523 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=977ms, rpcTimeout=969ms 2024-11-22T16:57:06,091 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=977ms, rpcTimeout=969ms 2024-11-22T16:57:06,091 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., hostname=4618c6346924,35523,1732294539277, seqNum=2 from cache 2024-11-22T16:57:06,100 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=352 (was 383), ProcessCount=11 (was 11), AvailableMemoryMB=7162 (was 7181) 2024-11-22T16:57:06,100 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:57:06,100 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:57:06,105 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:57:06,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:57:06,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T16:57:06,106 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T16:57:06,106 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:57:06,106 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:57:06,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:57:06,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:57:06,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T16:57:06,107 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T16:57:06,107 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1370267634, stopped=false 2024-11-22T16:57:06,107 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4618c6346924,38813,1732294538589 2024-11-22T16:57:06,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:57:06,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T16:57:06,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:57:06,109 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:57:06,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:57:06,110 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T16:57:06,110 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:57:06,110 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T16:57:06,110 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:57:06,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:57:06,111 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4618c6346924,35523,1732294539277' ***** 2024-11-22T16:57:06,111 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-22T16:57:06,111 INFO [RS:0;4618c6346924:35523 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T16:57:06,111 INFO [RS:0;4618c6346924:35523 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T16:57:06,111 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T16:57:06,111 INFO [RS:0;4618c6346924:35523 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T16:57:06,112 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 06d748ca8bbb56ba14ac063439728986 2024-11-22T16:57:06,112 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a7a1fb7610547cebdfcc43dd7732814 2024-11-22T16:57:06,112 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 23be4b4a4ddceb8e344defa4c78c05b3 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d84c6aa3b992bfbb823675e8d1ba55f 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 62ab92dff0aec162c9c270ccc6cfe4c0 2024-11-22T16:57:06,113 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06d748ca8bbb56ba14ac063439728986, disabling compactions & flushes 2024-11-22T16:57:06,113 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 5910e30db9d5827c0e378da14e38ccfa 2024-11-22T16:57:06,113 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 3edce1f8cf628b4ac8b7aa6984f09414 2024-11-22T16:57:06,113 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. after waiting 0 ms 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:06,113 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(959): stopping server 4618c6346924,35523,1732294539277 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:57:06,113 INFO [RS:0;4618c6346924:35523 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4618c6346924:35523. 2024-11-22T16:57:06,113 DEBUG [RS:0;4618c6346924:35523 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T16:57:06,113 DEBUG [RS:0;4618c6346924:35523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:57:06,114 INFO [RS:0;4618c6346924:35523 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T16:57:06,114 INFO [RS:0;4618c6346924:35523 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T16:57:06,114 INFO [RS:0;4618c6346924:35523 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T16:57:06,114 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T16:57:06,114 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-22T16:57:06,114 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T16:57:06,114 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T16:57:06,114 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T16:57:06,114 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T16:57:06,114 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T16:57:06,115 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 06d748ca8bbb56ba14ac063439728986, 1588230740, 1a7a1fb7610547cebdfcc43dd7732814, 20cea65d6b97f32bfc869ed22ded698a, 23be4b4a4ddceb8e344defa4c78c05b3, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:06,116 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 06d748ca8bbb56ba14ac063439728986 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:06,116 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-22T16:57:06,173 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986/.tmp/testFamily/c527e4e0e19d49e3a115f9c0c0422571 is 61, key is row-0/testFamily:testQualifier/1732294606922/Put/seqid=0 2024-11-22T16:57:06,181 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/info/63720d1741d74735a11c0c1bdbe9ea14 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814./info:regioninfo/1732294573669/Put/seqid=0 2024-11-22T16:57:06,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741855_1031 (size=5049) 2024-11-22T16:57:06,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741856_1032 (size=20564) 2024-11-22T16:57:06,315 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 06d748ca8bbb56ba14ac063439728986, 1588230740, 1a7a1fb7610547cebdfcc43dd7732814, 20cea65d6b97f32bfc869ed22ded698a, 23be4b4a4ddceb8e344defa4c78c05b3, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:06,515 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 06d748ca8bbb56ba14ac063439728986, 1588230740, 1a7a1fb7610547cebdfcc43dd7732814, 20cea65d6b97f32bfc869ed22ded698a, 23be4b4a4ddceb8e344defa4c78c05b3, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:06,584 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986/.tmp/testFamily/c527e4e0e19d49e3a115f9c0c0422571 2024-11-22T16:57:06,588 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/info/63720d1741d74735a11c0c1bdbe9ea14 2024-11-22T16:57:06,614 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-22T16:57:06,631 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986/.tmp/testFamily/c527e4e0e19d49e3a115f9c0c0422571 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986/testFamily/c527e4e0e19d49e3a115f9c0c0422571 2024-11-22T16:57:06,641 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986/testFamily/c527e4e0e19d49e3a115f9c0c0422571, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:06,647 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/ns/570605702cd74fa49708358b0ba8d36a is 43, key is default/ns:d/1732294572562/Put/seqid=0 2024-11-22T16:57:06,652 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 06d748ca8bbb56ba14ac063439728986 in 533ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:06,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741857_1033 (size=5153) 2024-11-22T16:57:06,658 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/06d748ca8bbb56ba14ac063439728986/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:06,661 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:57:06,661 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06d748ca8bbb56ba14ac063439728986: Waiting for close lock at 1732294626112Running coprocessor pre-close hooks at 1732294626113 (+1 ms)Disabling compacts and flushes for region at 1732294626113Disabling writes for close at 1732294626113Obtaining lock to block concurrent updates at 1732294626116 (+3 ms)Preparing flush snapshotting stores in 06d748ca8bbb56ba14ac063439728986 at 1732294626116Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294626124 (+8 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. at 1732294626126 (+2 ms)Flushing 06d748ca8bbb56ba14ac063439728986/testFamily: creating writer at 1732294626127 (+1 ms)Flushing 06d748ca8bbb56ba14ac063439728986/testFamily: appending metadata at 1732294626168 (+41 ms)Flushing 06d748ca8bbb56ba14ac063439728986/testFamily: closing flushed file at 1732294626170 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@550a810d: reopening flushed file at 1732294626630 (+460 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 06d748ca8bbb56ba14ac063439728986 in 533ms, sequenceid=8, compaction requested=false at 1732294626652 (+22 ms)Writing region close event to WAL at 1732294626653 (+1 ms)Running coprocessor post-close hooks at 1732294626659 (+6 ms)Closed at 1732294626660 (+1 ms) 2024-11-22T16:57:06,661 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732294605885.06d748ca8bbb56ba14ac063439728986. 2024-11-22T16:57:06,661 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a7a1fb7610547cebdfcc43dd7732814, disabling compactions & flushes 2024-11-22T16:57:06,662 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:57:06,662 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:57:06,662 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. after waiting 0 ms 2024-11-22T16:57:06,662 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:57:06,662 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1a7a1fb7610547cebdfcc43dd7732814 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:06,678 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T16:57:06,681 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814/.tmp/testFamily/c1281aeec6ba48cbbcc3bf191164c550 is 61, key is row-0/testFamily:testQualifier/1732294574121/Put/seqid=0 2024-11-22T16:57:06,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741858_1034 (size=5049) 2024-11-22T16:57:06,704 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-22T16:57:06,716 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a7a1fb7610547cebdfcc43dd7732814, 20cea65d6b97f32bfc869ed22ded698a, 23be4b4a4ddceb8e344defa4c78c05b3, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:06,759 INFO [regionserver/4618c6346924:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T16:57:06,759 INFO [regionserver/4618c6346924:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T16:57:06,916 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a7a1fb7610547cebdfcc43dd7732814, 20cea65d6b97f32bfc869ed22ded698a, 23be4b4a4ddceb8e344defa4c78c05b3, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:07,053 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/ns/570605702cd74fa49708358b0ba8d36a 2024-11-22T16:57:07,077 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/table/3c6eb45587f84a4dac54ea44b8c30056 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732294573703/Put/seqid=0 2024-11-22T16:57:07,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741859_1035 (size=6699) 2024-11-22T16:57:07,087 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814/.tmp/testFamily/c1281aeec6ba48cbbcc3bf191164c550 2024-11-22T16:57:07,096 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814/.tmp/testFamily/c1281aeec6ba48cbbcc3bf191164c550 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814/testFamily/c1281aeec6ba48cbbcc3bf191164c550 2024-11-22T16:57:07,104 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814/testFamily/c1281aeec6ba48cbbcc3bf191164c550, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:07,105 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1a7a1fb7610547cebdfcc43dd7732814 in 443ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:07,110 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextExceptionAsync/1a7a1fb7610547cebdfcc43dd7732814/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:07,111 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:57:07,111 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a7a1fb7610547cebdfcc43dd7732814: Waiting for close lock at 1732294626661Running coprocessor pre-close hooks at 1732294626661Disabling compacts and flushes for region at 1732294626661Disabling writes for close at 1732294626662 (+1 ms)Obtaining lock to block concurrent updates at 1732294626662Preparing flush snapshotting stores in 1a7a1fb7610547cebdfcc43dd7732814 at 1732294626662Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294626662Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. at 1732294626663 (+1 ms)Flushing 1a7a1fb7610547cebdfcc43dd7732814/testFamily: creating writer at 1732294626663Flushing 1a7a1fb7610547cebdfcc43dd7732814/testFamily: appending metadata at 1732294626681 (+18 ms)Flushing 1a7a1fb7610547cebdfcc43dd7732814/testFamily: closing flushed file at 1732294626681Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2829085f: reopening flushed file at 1732294627095 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1a7a1fb7610547cebdfcc43dd7732814 in 443ms, sequenceid=8, compaction requested=false at 1732294627105 (+10 ms)Writing region close event to WAL at 1732294627106 (+1 ms)Running coprocessor post-close hooks at 1732294627111 (+5 ms)Closed at 1732294627111 2024-11-22T16:57:07,111 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732294572943.1a7a1fb7610547cebdfcc43dd7732814. 2024-11-22T16:57:07,111 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23be4b4a4ddceb8e344defa4c78c05b3, disabling compactions & flushes 2024-11-22T16:57:07,111 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:57:07,111 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:57:07,111 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. after waiting 0 ms 2024-11-22T16:57:07,112 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:57:07,112 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 23be4b4a4ddceb8e344defa4c78c05b3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:07,116 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-22T16:57:07,116 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1325): Online Regions={23be4b4a4ddceb8e344defa4c78c05b3=hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3., d3d4d5f2feb4a5ec59970ac2ede0a2ce=hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce., 1588230740=hbase:meta,,1.1588230740, 2d84c6aa3b992bfbb823675e8d1ba55f=testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f., 62ab92dff0aec162c9c270ccc6cfe4c0=hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., 5910e30db9d5827c0e378da14e38ccfa=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., 3edce1f8cf628b4ac8b7aa6984f09414=testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414., 744b5f7fd7cfd3ba3e97d73d358d5754=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., 20cea65d6b97f32bfc869ed22ded698a=testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.} 2024-11-22T16:57:07,116 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 20cea65d6b97f32bfc869ed22ded698a, 23be4b4a4ddceb8e344defa4c78c05b3, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:07,130 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3/.tmp/testFamily/1bd584b7f7b844ce89d5db7b2c2b4ffd is 61, key is row-0/testFamily:testQualifier/1732294582485/Put/seqid=0 2024-11-22T16:57:07,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741860_1036 (size=5049) 2024-11-22T16:57:07,317 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 20cea65d6b97f32bfc869ed22ded698a, 23be4b4a4ddceb8e344defa4c78c05b3, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:07,483 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/table/3c6eb45587f84a4dac54ea44b8c30056 2024-11-22T16:57:07,492 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/info/63720d1741d74735a11c0c1bdbe9ea14 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/info/63720d1741d74735a11c0c1bdbe9ea14 2024-11-22T16:57:07,499 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/info/63720d1741d74735a11c0c1bdbe9ea14, entries=100, sequenceid=56, filesize=20.1 K 2024-11-22T16:57:07,501 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/ns/570605702cd74fa49708358b0ba8d36a as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/ns/570605702cd74fa49708358b0ba8d36a 2024-11-22T16:57:07,508 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/ns/570605702cd74fa49708358b0ba8d36a, entries=2, sequenceid=56, filesize=5.0 K 2024-11-22T16:57:07,510 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/.tmp/table/3c6eb45587f84a4dac54ea44b8c30056 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/table/3c6eb45587f84a4dac54ea44b8c30056 2024-11-22T16:57:07,517 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 20cea65d6b97f32bfc869ed22ded698a, 23be4b4a4ddceb8e344defa4c78c05b3, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:07,518 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/table/3c6eb45587f84a4dac54ea44b8c30056, entries=20, sequenceid=56, filesize=6.5 K 2024-11-22T16:57:07,520 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1405ms, sequenceid=56, compaction requested=false 2024-11-22T16:57:07,525 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-22T16:57:07,526 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T16:57:07,526 INFO [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T16:57:07,527 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732294626114Running coprocessor pre-close hooks at 1732294626114Disabling compacts and flushes for region at 1732294626114Disabling writes for close at 1732294626114Obtaining lock to block concurrent updates at 1732294626116 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732294626116Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732294626124 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732294626126 (+2 ms)Flushing 1588230740/info: creating writer at 1732294626127 (+1 ms)Flushing 1588230740/info: appending metadata at 1732294626176 (+49 ms)Flushing 1588230740/info: closing flushed file at 1732294626176Flushing 1588230740/ns: creating writer at 1732294626630 (+454 ms)Flushing 1588230740/ns: appending metadata at 1732294626646 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732294626646Flushing 1588230740/table: creating writer at 1732294627061 (+415 ms)Flushing 1588230740/table: appending metadata at 1732294627076 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732294627076Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46328f50: reopening flushed file at 1732294627491 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73a6a118: reopening flushed file at 1732294627500 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37f64e2e: reopening flushed file at 1732294627509 (+9 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1405ms, sequenceid=56, compaction requested=false at 1732294627520 (+11 ms)Writing region close event to WAL at 1732294627521 (+1 ms)Running coprocessor post-close hooks at 1732294627526 (+5 ms)Closed at 1732294627526 2024-11-22T16:57:07,527 DEBUG [RS_CLOSE_META-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T16:57:07,537 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3/.tmp/testFamily/1bd584b7f7b844ce89d5db7b2c2b4ffd 2024-11-22T16:57:07,545 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3/.tmp/testFamily/1bd584b7f7b844ce89d5db7b2c2b4ffd as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3/testFamily/1bd584b7f7b844ce89d5db7b2c2b4ffd 2024-11-22T16:57:07,553 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3/testFamily/1bd584b7f7b844ce89d5db7b2c2b4ffd, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:07,554 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 23be4b4a4ddceb8e344defa4c78c05b3 in 442ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:07,558 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNextAsync/23be4b4a4ddceb8e344defa4c78c05b3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:07,559 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:57:07,559 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23be4b4a4ddceb8e344defa4c78c05b3: Waiting for close lock at 1732294627111Running coprocessor pre-close hooks at 1732294627111Disabling compacts and flushes for region at 1732294627111Disabling writes for close at 1732294627112 (+1 ms)Obtaining lock to block concurrent updates at 1732294627112Preparing flush snapshotting stores in 23be4b4a4ddceb8e344defa4c78c05b3 at 1732294627112Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294627112Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. at 1732294627113 (+1 ms)Flushing 23be4b4a4ddceb8e344defa4c78c05b3/testFamily: creating writer at 1732294627113Flushing 23be4b4a4ddceb8e344defa4c78c05b3/testFamily: appending metadata at 1732294627130 (+17 ms)Flushing 23be4b4a4ddceb8e344defa4c78c05b3/testFamily: closing flushed file at 1732294627130Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13043696: reopening flushed file at 1732294627544 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 23be4b4a4ddceb8e344defa4c78c05b3 in 442ms, sequenceid=8, compaction requested=false at 1732294627554 (+10 ms)Writing region close event to WAL at 1732294627555 (+1 ms)Running coprocessor post-close hooks at 1732294627559 (+4 ms)Closed at 1732294627559 2024-11-22T16:57:07,559 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732294581448.23be4b4a4ddceb8e344defa4c78c05b3. 2024-11-22T16:57:07,560 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d3d4d5f2feb4a5ec59970ac2ede0a2ce, disabling compactions & flushes 2024-11-22T16:57:07,560 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:57:07,560 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:57:07,560 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. after waiting 0 ms 2024-11-22T16:57:07,560 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:57:07,560 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d3d4d5f2feb4a5ec59970ac2ede0a2ce 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:07,577 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce/.tmp/testFamily/8389f4a091514bb68a85fe940d1cb18f is 61, key is row-0/testFamily:testQualifier/1732294613990/Put/seqid=0 2024-11-22T16:57:07,581 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-22T16:57:07,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741861_1037 (size=5049) 2024-11-22T16:57:07,717 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:07,917 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754, d3d4d5f2feb4a5ec59970ac2ede0a2ce 2024-11-22T16:57:07,982 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce/.tmp/testFamily/8389f4a091514bb68a85fe940d1cb18f 2024-11-22T16:57:07,991 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce/.tmp/testFamily/8389f4a091514bb68a85fe940d1cb18f as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce/testFamily/8389f4a091514bb68a85fe940d1cb18f 2024-11-22T16:57:07,998 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce/testFamily/8389f4a091514bb68a85fe940d1cb18f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:07,999 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d3d4d5f2feb4a5ec59970ac2ede0a2ce in 439ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:08,003 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnNext/d3d4d5f2feb4a5ec59970ac2ede0a2ce/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:08,004 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:57:08,004 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d3d4d5f2feb4a5ec59970ac2ede0a2ce: Waiting for close lock at 1732294627560Running coprocessor pre-close hooks at 1732294627560Disabling compacts and flushes for region at 1732294627560Disabling writes for close at 1732294627560Obtaining lock to block concurrent updates at 1732294627560Preparing flush snapshotting stores in d3d4d5f2feb4a5ec59970ac2ede0a2ce at 1732294627560Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294627560Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. at 1732294627561 (+1 ms)Flushing d3d4d5f2feb4a5ec59970ac2ede0a2ce/testFamily: creating writer at 1732294627561Flushing d3d4d5f2feb4a5ec59970ac2ede0a2ce/testFamily: appending metadata at 1732294627577 (+16 ms)Flushing d3d4d5f2feb4a5ec59970ac2ede0a2ce/testFamily: closing flushed file at 1732294627577Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c1c18a4: reopening flushed file at 1732294627990 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d3d4d5f2feb4a5ec59970ac2ede0a2ce in 439ms, sequenceid=8, compaction requested=false at 1732294627999 (+9 ms)Writing region close event to WAL at 1732294627999Running coprocessor post-close hooks at 1732294628004 (+5 ms)Closed at 1732294628004 2024-11-22T16:57:08,004 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732294612966.d3d4d5f2feb4a5ec59970ac2ede0a2ce. 2024-11-22T16:57:08,004 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d84c6aa3b992bfbb823675e8d1ba55f, disabling compactions & flushes 2024-11-22T16:57:08,004 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:57:08,004 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:57:08,004 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. after waiting 0 ms 2024-11-22T16:57:08,004 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:57:08,004 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2d84c6aa3b992bfbb823675e8d1ba55f 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:08,021 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f/.tmp/testFamily/ec688cbacf43459cb2be31d2548c8e0d is 61, key is row-0/testFamily:testQualifier/1732294598770/Put/seqid=0 2024-11-22T16:57:08,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741862_1038 (size=5049) 2024-11-22T16:57:08,118 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-22T16:57:08,118 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1325): Online Regions={2d84c6aa3b992bfbb823675e8d1ba55f=testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f., 62ab92dff0aec162c9c270ccc6cfe4c0=hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., 5910e30db9d5827c0e378da14e38ccfa=testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., 3edce1f8cf628b4ac8b7aa6984f09414=testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414., 744b5f7fd7cfd3ba3e97d73d358d5754=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., 20cea65d6b97f32bfc869ed22ded698a=testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.} 2024-11-22T16:57:08,118 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:08,318 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 2d84c6aa3b992bfbb823675e8d1ba55f, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 62ab92dff0aec162c9c270ccc6cfe4c0, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:08,427 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f/.tmp/testFamily/ec688cbacf43459cb2be31d2548c8e0d 2024-11-22T16:57:08,438 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f/.tmp/testFamily/ec688cbacf43459cb2be31d2548c8e0d as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f/testFamily/ec688cbacf43459cb2be31d2548c8e0d 2024-11-22T16:57:08,446 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f/testFamily/ec688cbacf43459cb2be31d2548c8e0d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:08,447 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2d84c6aa3b992bfbb823675e8d1ba55f in 443ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:08,453 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNextAsync/2d84c6aa3b992bfbb823675e8d1ba55f/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:08,454 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:57:08,454 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d84c6aa3b992bfbb823675e8d1ba55f: Waiting for close lock at 1732294628004Running coprocessor pre-close hooks at 1732294628004Disabling compacts and flushes for region at 1732294628004Disabling writes for close at 1732294628004Obtaining lock to block concurrent updates at 1732294628005 (+1 ms)Preparing flush snapshotting stores in 2d84c6aa3b992bfbb823675e8d1ba55f at 1732294628005Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294628005Flushing stores of testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. at 1732294628005Flushing 2d84c6aa3b992bfbb823675e8d1ba55f/testFamily: creating writer at 1732294628005Flushing 2d84c6aa3b992bfbb823675e8d1ba55f/testFamily: appending metadata at 1732294628020 (+15 ms)Flushing 2d84c6aa3b992bfbb823675e8d1ba55f/testFamily: closing flushed file at 1732294628020Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20debfd1: reopening flushed file at 1732294628436 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2d84c6aa3b992bfbb823675e8d1ba55f in 443ms, sequenceid=8, compaction requested=false at 1732294628448 (+12 ms)Writing region close event to WAL at 1732294628448Running coprocessor post-close hooks at 1732294628454 (+6 ms)Closed at 1732294628454 2024-11-22T16:57:08,454 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732294595711.2d84c6aa3b992bfbb823675e8d1ba55f. 2024-11-22T16:57:08,454 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 62ab92dff0aec162c9c270ccc6cfe4c0, disabling compactions & flushes 2024-11-22T16:57:08,454 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:57:08,454 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:57:08,454 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. after waiting 0 ms 2024-11-22T16:57:08,455 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:57:08,455 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 62ab92dff0aec162c9c270ccc6cfe4c0 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:08,477 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0/.tmp/testFamily/328e6780866045c4b7baee328d76b353 is 61, key is row-0/testFamily:testQualifier/1732294575383/Put/seqid=0 2024-11-22T16:57:08,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741863_1039 (size=5049) 2024-11-22T16:57:08,483 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0/.tmp/testFamily/328e6780866045c4b7baee328d76b353 2024-11-22T16:57:08,493 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0/.tmp/testFamily/328e6780866045c4b7baee328d76b353 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0/testFamily/328e6780866045c4b7baee328d76b353 2024-11-22T16:57:08,503 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0/testFamily/328e6780866045c4b7baee328d76b353, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:08,506 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 62ab92dff0aec162c9c270ccc6cfe4c0 in 51ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:08,512 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/hbase/testMetaScanTimeoutOnOpenScanner/62ab92dff0aec162c9c270ccc6cfe4c0/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:08,513 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:57:08,513 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 62ab92dff0aec162c9c270ccc6cfe4c0: Waiting for close lock at 1732294628454Running coprocessor pre-close hooks at 1732294628454Disabling compacts and flushes for region at 1732294628454Disabling writes for close at 1732294628455 (+1 ms)Obtaining lock to block concurrent updates at 1732294628455Preparing flush snapshotting stores in 62ab92dff0aec162c9c270ccc6cfe4c0 at 1732294628455Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294628455Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. at 1732294628456 (+1 ms)Flushing 62ab92dff0aec162c9c270ccc6cfe4c0/testFamily: creating writer at 1732294628456Flushing 62ab92dff0aec162c9c270ccc6cfe4c0/testFamily: appending metadata at 1732294628476 (+20 ms)Flushing 62ab92dff0aec162c9c270ccc6cfe4c0/testFamily: closing flushed file at 1732294628476Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2db14d15: reopening flushed file at 1732294628491 (+15 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 62ab92dff0aec162c9c270ccc6cfe4c0 in 51ms, sequenceid=8, compaction requested=false at 1732294628506 (+15 ms)Writing region close event to WAL at 1732294628507 (+1 ms)Running coprocessor post-close hooks at 1732294628513 (+6 ms)Closed at 1732294628513 2024-11-22T16:57:08,513 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732294574350.62ab92dff0aec162c9c270ccc6cfe4c0. 2024-11-22T16:57:08,513 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5910e30db9d5827c0e378da14e38ccfa, disabling compactions & flushes 2024-11-22T16:57:08,513 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:57:08,513 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:57:08,514 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. after waiting 0 ms 2024-11-22T16:57:08,514 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:57:08,514 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5910e30db9d5827c0e378da14e38ccfa 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:08,518 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:08,538 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa/.tmp/testFamily/7d32bdd86c3b488d8bc0c09129f07c2a is 61, key is row-0/testFamily:testQualifier/1732294592670/Put/seqid=0 2024-11-22T16:57:08,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741864_1040 (size=5049) 2024-11-22T16:57:08,719 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:08,919 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 3edce1f8cf628b4ac8b7aa6984f09414, 5910e30db9d5827c0e378da14e38ccfa, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:08,944 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa/.tmp/testFamily/7d32bdd86c3b488d8bc0c09129f07c2a 2024-11-22T16:57:08,954 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa/.tmp/testFamily/7d32bdd86c3b488d8bc0c09129f07c2a as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa/testFamily/7d32bdd86c3b488d8bc0c09129f07c2a 2024-11-22T16:57:08,966 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa/testFamily/7d32bdd86c3b488d8bc0c09129f07c2a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:08,967 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5910e30db9d5827c0e378da14e38ccfa in 453ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:08,972 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScanner/5910e30db9d5827c0e378da14e38ccfa/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:08,973 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:57:08,973 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5910e30db9d5827c0e378da14e38ccfa: Waiting for close lock at 1732294628513Running coprocessor pre-close hooks at 1732294628513Disabling compacts and flushes for region at 1732294628513Disabling writes for close at 1732294628514 (+1 ms)Obtaining lock to block concurrent updates at 1732294628514Preparing flush snapshotting stores in 5910e30db9d5827c0e378da14e38ccfa at 1732294628514Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294628514Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. at 1732294628515 (+1 ms)Flushing 5910e30db9d5827c0e378da14e38ccfa/testFamily: creating writer at 1732294628515Flushing 5910e30db9d5827c0e378da14e38ccfa/testFamily: appending metadata at 1732294628537 (+22 ms)Flushing 5910e30db9d5827c0e378da14e38ccfa/testFamily: closing flushed file at 1732294628537Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71405c68: reopening flushed file at 1732294628953 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5910e30db9d5827c0e378da14e38ccfa in 453ms, sequenceid=8, compaction requested=false at 1732294628967 (+14 ms)Writing region close event to WAL at 1732294628968 (+1 ms)Running coprocessor post-close hooks at 1732294628973 (+5 ms)Closed at 1732294628973 2024-11-22T16:57:08,973 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732294591641.5910e30db9d5827c0e378da14e38ccfa. 2024-11-22T16:57:08,974 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3edce1f8cf628b4ac8b7aa6984f09414, disabling compactions & flushes 2024-11-22T16:57:08,974 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:57:08,974 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:57:08,974 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. after waiting 0 ms 2024-11-22T16:57:08,974 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:57:08,974 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3edce1f8cf628b4ac8b7aa6984f09414 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:08,999 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414/.tmp/testFamily/f52cfae96cf842d4a9e2e1618dedfc83 is 61, key is row-0/testFamily:testQualifier/1732294602842/Put/seqid=0 2024-11-22T16:57:09,002 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-22T16:57:09,002 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-22T16:57:09,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-22T16:57:09,003 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-22T16:57:09,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-22T16:57:09,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-22T16:57:09,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T16:57:09,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-22T16:57:09,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-22T16:57:09,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-22T16:57:09,004 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-22T16:57:09,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741865_1041 (size=5049) 2024-11-22T16:57:09,119 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-22T16:57:09,119 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1325): Online Regions={3edce1f8cf628b4ac8b7aa6984f09414=testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414., 744b5f7fd7cfd3ba3e97d73d358d5754=testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., 20cea65d6b97f32bfc869ed22ded698a=testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.} 2024-11-22T16:57:09,120 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 3edce1f8cf628b4ac8b7aa6984f09414, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:09,320 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 3edce1f8cf628b4ac8b7aa6984f09414, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:09,410 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414/.tmp/testFamily/f52cfae96cf842d4a9e2e1618dedfc83 2024-11-22T16:57:09,423 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414/.tmp/testFamily/f52cfae96cf842d4a9e2e1618dedfc83 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414/testFamily/f52cfae96cf842d4a9e2e1618dedfc83 2024-11-22T16:57:09,434 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414/testFamily/f52cfae96cf842d4a9e2e1618dedfc83, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:09,436 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3edce1f8cf628b4ac8b7aa6984f09414 in 462ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:09,442 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnNext/3edce1f8cf628b4ac8b7aa6984f09414/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:09,443 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:57:09,443 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3edce1f8cf628b4ac8b7aa6984f09414: Waiting for close lock at 1732294628974Running coprocessor pre-close hooks at 1732294628974Disabling compacts and flushes for region at 1732294628974Disabling writes for close at 1732294628974Obtaining lock to block concurrent updates at 1732294628974Preparing flush snapshotting stores in 3edce1f8cf628b4ac8b7aa6984f09414 at 1732294628974Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294628974Flushing stores of testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. at 1732294628975 (+1 ms)Flushing 3edce1f8cf628b4ac8b7aa6984f09414/testFamily: creating writer at 1732294628976 (+1 ms)Flushing 3edce1f8cf628b4ac8b7aa6984f09414/testFamily: appending metadata at 1732294628998 (+22 ms)Flushing 3edce1f8cf628b4ac8b7aa6984f09414/testFamily: closing flushed file at 1732294628998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2002edf6: reopening flushed file at 1732294629422 (+424 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3edce1f8cf628b4ac8b7aa6984f09414 in 462ms, sequenceid=8, compaction requested=false at 1732294629436 (+14 ms)Writing region close event to WAL at 1732294629437 (+1 ms)Running coprocessor post-close hooks at 1732294629443 (+6 ms)Closed at 1732294629443 2024-11-22T16:57:09,443 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732294601816.3edce1f8cf628b4ac8b7aa6984f09414. 2024-11-22T16:57:09,443 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 744b5f7fd7cfd3ba3e97d73d358d5754, disabling compactions & flushes 2024-11-22T16:57:09,443 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:09,443 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:09,444 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. after waiting 0 ms 2024-11-22T16:57:09,444 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:09,444 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 744b5f7fd7cfd3ba3e97d73d358d5754 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:09,468 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754/.tmp/testFamily/03662fcb62704ce1910df6dd0b56eb99 is 61, key is row-0/testFamily:testQualifier/1732294623070/Put/seqid=0 2024-11-22T16:57:09,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741866_1042 (size=5049) 2024-11-22T16:57:09,520 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:09,720 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a, 744b5f7fd7cfd3ba3e97d73d358d5754 2024-11-22T16:57:09,877 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754/.tmp/testFamily/03662fcb62704ce1910df6dd0b56eb99 2024-11-22T16:57:09,888 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754/.tmp/testFamily/03662fcb62704ce1910df6dd0b56eb99 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754/testFamily/03662fcb62704ce1910df6dd0b56eb99 2024-11-22T16:57:09,898 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754/testFamily/03662fcb62704ce1910df6dd0b56eb99, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:09,899 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 744b5f7fd7cfd3ba3e97d73d358d5754 in 455ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:09,906 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testNormalScanTimeoutOnOpenScannerAsync/744b5f7fd7cfd3ba3e97d73d358d5754/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:09,907 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:09,907 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 744b5f7fd7cfd3ba3e97d73d358d5754: Waiting for close lock at 1732294629443Running coprocessor pre-close hooks at 1732294629443Disabling compacts and flushes for region at 1732294629443Disabling writes for close at 1732294629444 (+1 ms)Obtaining lock to block concurrent updates at 1732294629444Preparing flush snapshotting stores in 744b5f7fd7cfd3ba3e97d73d358d5754 at 1732294629444Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294629444Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. at 1732294629445 (+1 ms)Flushing 744b5f7fd7cfd3ba3e97d73d358d5754/testFamily: creating writer at 1732294629445Flushing 744b5f7fd7cfd3ba3e97d73d358d5754/testFamily: appending metadata at 1732294629468 (+23 ms)Flushing 744b5f7fd7cfd3ba3e97d73d358d5754/testFamily: closing flushed file at 1732294629468Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c703525: reopening flushed file at 1732294629886 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 744b5f7fd7cfd3ba3e97d73d358d5754 in 455ms, sequenceid=8, compaction requested=false at 1732294629899 (+13 ms)Writing region close event to WAL at 1732294629900 (+1 ms)Running coprocessor post-close hooks at 1732294629906 (+6 ms)Closed at 1732294629907 (+1 ms) 2024-11-22T16:57:09,907 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732294620030.744b5f7fd7cfd3ba3e97d73d358d5754. 2024-11-22T16:57:09,907 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 20cea65d6b97f32bfc869ed22ded698a, disabling compactions & flushes 2024-11-22T16:57:09,907 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:57:09,907 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:57:09,907 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. after waiting 0 ms 2024-11-22T16:57:09,907 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:57:09,907 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 20cea65d6b97f32bfc869ed22ded698a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-22T16:57:09,921 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:57:09,925 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a/.tmp/testFamily/85ded187d05c47ddb48897e6f806b113 is 61, key is row-0/testFamily:testQualifier/1732294591572/Put/seqid=0 2024-11-22T16:57:09,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741867_1043 (size=5049) 2024-11-22T16:57:10,121 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-22T16:57:10,121 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1325): Online Regions={20cea65d6b97f32bfc869ed22ded698a=testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a.} 2024-11-22T16:57:10,121 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:57:10,322 DEBUG [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1351): Waiting on 20cea65d6b97f32bfc869ed22ded698a 2024-11-22T16:57:10,333 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a/.tmp/testFamily/85ded187d05c47ddb48897e6f806b113 2024-11-22T16:57:10,344 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a/.tmp/testFamily/85ded187d05c47ddb48897e6f806b113 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a/testFamily/85ded187d05c47ddb48897e6f806b113 2024-11-22T16:57:10,352 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a/testFamily/85ded187d05c47ddb48897e6f806b113, entries=4, sequenceid=8, filesize=4.9 K 2024-11-22T16:57:10,354 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 20cea65d6b97f32bfc869ed22ded698a in 447ms, sequenceid=8, compaction requested=false 2024-11-22T16:57:10,362 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/data/default/testRetryOutOfOrderScannerNextException/20cea65d6b97f32bfc869ed22ded698a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-22T16:57:10,363 INFO [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:57:10,363 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 20cea65d6b97f32bfc869ed22ded698a: Waiting for close lock at 1732294629907Running coprocessor pre-close hooks at 1732294629907Disabling compacts and flushes for region at 1732294629907Disabling writes for close at 1732294629907Obtaining lock to block concurrent updates at 1732294629907Preparing flush snapshotting stores in 20cea65d6b97f32bfc869ed22ded698a at 1732294629907Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732294629908 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. at 1732294629908Flushing 20cea65d6b97f32bfc869ed22ded698a/testFamily: creating writer at 1732294629908Flushing 20cea65d6b97f32bfc869ed22ded698a/testFamily: appending metadata at 1732294629925 (+17 ms)Flushing 20cea65d6b97f32bfc869ed22ded698a/testFamily: closing flushed file at 1732294629925Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21f1c3cd: reopening flushed file at 1732294630343 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 20cea65d6b97f32bfc869ed22ded698a in 447ms, sequenceid=8, compaction requested=false at 1732294630354 (+11 ms)Writing region close event to WAL at 1732294630355 (+1 ms)Running coprocessor post-close hooks at 1732294630363 (+8 ms)Closed at 1732294630363 2024-11-22T16:57:10,363 DEBUG [RS_CLOSE_REGION-regionserver/4618c6346924:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732294588537.20cea65d6b97f32bfc869ed22ded698a. 2024-11-22T16:57:10,522 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(976): stopping server 4618c6346924,35523,1732294539277; all regions closed. 2024-11-22T16:57:10,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741834_1010 (size=19479) 2024-11-22T16:57:10,534 DEBUG [RS:0;4618c6346924:35523 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/oldWALs 2024-11-22T16:57:10,534 INFO [RS:0;4618c6346924:35523 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C35523%2C1732294539277.meta:.meta(num 1732294572271) 2024-11-22T16:57:10,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741833_1009 (size=20031) 2024-11-22T16:57:10,540 DEBUG [RS:0;4618c6346924:35523 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/oldWALs 2024-11-22T16:57:10,540 INFO [RS:0;4618c6346924:35523 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4618c6346924%2C35523%2C1732294539277:(num 1732294571817) 2024-11-22T16:57:10,540 DEBUG [RS:0;4618c6346924:35523 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T16:57:10,540 INFO [RS:0;4618c6346924:35523 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:57:10,541 INFO [RS:0;4618c6346924:35523 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:57:10,541 INFO [RS:0;4618c6346924:35523 {}] hbase.ChoreService(370): Chore service for: regionserver/4618c6346924:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-22T16:57:10,541 INFO [RS:0;4618c6346924:35523 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:57:10,541 INFO [regionserver/4618c6346924:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:57:10,542 INFO [RS:0;4618c6346924:35523 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35523 2024-11-22T16:57:10,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T16:57:10,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4618c6346924,35523,1732294539277 2024-11-22T16:57:10,547 INFO [RS:0;4618c6346924:35523 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:57:10,548 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4618c6346924,35523,1732294539277] 2024-11-22T16:57:10,551 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4618c6346924,35523,1732294539277 already deleted, retry=false 2024-11-22T16:57:10,551 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4618c6346924,35523,1732294539277 expired; onlineServers=0 2024-11-22T16:57:10,551 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4618c6346924,38813,1732294538589' ***** 2024-11-22T16:57:10,551 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-22T16:57:10,551 INFO [M:0;4618c6346924:38813 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T16:57:10,551 INFO [M:0;4618c6346924:38813 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T16:57:10,551 DEBUG [M:0;4618c6346924:38813 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T16:57:10,552 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T16:57:10,552 DEBUG [M:0;4618c6346924:38813 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T16:57:10,552 DEBUG [master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732294541640 {}] cleaner.HFileCleaner(306): Exit Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.large.0-1732294541640,5,FailOnTimeoutGroup] 2024-11-22T16:57:10,552 DEBUG [master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732294541641 {}] cleaner.HFileCleaner(306): Exit Thread[master/4618c6346924:0:becomeActiveMaster-HFileCleaner.small.0-1732294541641,5,FailOnTimeoutGroup] 2024-11-22T16:57:10,552 INFO [M:0;4618c6346924:38813 {}] hbase.ChoreService(370): Chore service for: master/4618c6346924:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T16:57:10,552 INFO [M:0;4618c6346924:38813 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T16:57:10,552 DEBUG [M:0;4618c6346924:38813 {}] master.HMaster(1795): Stopping service threads 2024-11-22T16:57:10,553 INFO [M:0;4618c6346924:38813 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T16:57:10,553 INFO [M:0;4618c6346924:38813 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T16:57:10,553 INFO [M:0;4618c6346924:38813 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T16:57:10,554 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T16:57:10,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T16:57:10,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T16:57:10,555 DEBUG [M:0;4618c6346924:38813 {}] zookeeper.ZKUtil(347): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T16:57:10,556 WARN [M:0;4618c6346924:38813 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T16:57:10,557 INFO [M:0;4618c6346924:38813 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/.lastflushedseqids 2024-11-22T16:57:10,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741868_1044 (size=460) 2024-11-22T16:57:10,571 INFO [M:0;4618c6346924:38813 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T16:57:10,571 INFO [regionserver/4618c6346924:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T16:57:10,571 INFO [M:0;4618c6346924:38813 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T16:57:10,572 DEBUG [M:0;4618c6346924:38813 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T16:57:10,572 INFO [M:0;4618c6346924:38813 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:57:10,572 DEBUG [M:0;4618c6346924:38813 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:57:10,572 DEBUG [M:0;4618c6346924:38813 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T16:57:10,572 DEBUG [M:0;4618c6346924:38813 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:57:10,572 INFO [M:0;4618c6346924:38813 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.91 KB heapSize=190.38 KB 2024-11-22T16:57:10,592 DEBUG [M:0;4618c6346924:38813 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c5959fc657474f82a88b93ea49fad326 is 82, key is hbase:meta,,1/info:regioninfo/1732294572453/Put/seqid=0 2024-11-22T16:57:10,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741869_1045 (size=5672) 2024-11-22T16:57:10,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:57:10,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35523-0x101409adc3d0001, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:57:10,650 INFO [RS:0;4618c6346924:35523 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:57:10,650 INFO [RS:0;4618c6346924:35523 {}] regionserver.HRegionServer(1031): Exiting; stopping=4618c6346924,35523,1732294539277; zookeeper connection closed. 2024-11-22T16:57:10,651 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@739e5531 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@739e5531 2024-11-22T16:57:10,651 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T16:57:10,999 INFO [M:0;4618c6346924:38813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c5959fc657474f82a88b93ea49fad326 2024-11-22T16:57:11,035 DEBUG [M:0;4618c6346924:38813 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0369cc25116e4d54a14b16ba41131e28 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732294573714/Put/seqid=0 2024-11-22T16:57:11,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741870_1046 (size=14079) 2024-11-22T16:57:11,443 INFO [M:0;4618c6346924:38813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.35 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0369cc25116e4d54a14b16ba41131e28 2024-11-22T16:57:11,451 INFO [M:0;4618c6346924:38813 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0369cc25116e4d54a14b16ba41131e28 2024-11-22T16:57:11,479 DEBUG [M:0;4618c6346924:38813 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1585720777754dc998983f41e9454bcf is 69, key is 4618c6346924,35523,1732294539277/rs:state/1732294541677/Put/seqid=0 2024-11-22T16:57:11,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741871_1047 (size=5156) 2024-11-22T16:57:11,485 INFO [M:0;4618c6346924:38813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1585720777754dc998983f41e9454bcf 2024-11-22T16:57:11,494 DEBUG [M:0;4618c6346924:38813 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c5959fc657474f82a88b93ea49fad326 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c5959fc657474f82a88b93ea49fad326 2024-11-22T16:57:11,502 INFO [M:0;4618c6346924:38813 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c5959fc657474f82a88b93ea49fad326, entries=8, sequenceid=311, filesize=5.5 K 2024-11-22T16:57:11,503 DEBUG [M:0;4618c6346924:38813 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0369cc25116e4d54a14b16ba41131e28 as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0369cc25116e4d54a14b16ba41131e28 2024-11-22T16:57:11,509 INFO [M:0;4618c6346924:38813 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0369cc25116e4d54a14b16ba41131e28 2024-11-22T16:57:11,509 INFO [M:0;4618c6346924:38813 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0369cc25116e4d54a14b16ba41131e28, entries=33, sequenceid=311, filesize=13.7 K 2024-11-22T16:57:11,511 DEBUG [M:0;4618c6346924:38813 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1585720777754dc998983f41e9454bcf as hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1585720777754dc998983f41e9454bcf 2024-11-22T16:57:11,518 INFO [M:0;4618c6346924:38813 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35361/user/jenkins/test-data/c34fe022-e36c-22e2-df55-6e736297484d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1585720777754dc998983f41e9454bcf, entries=1, sequenceid=311, filesize=5.0 K 2024-11-22T16:57:11,520 INFO [M:0;4618c6346924:38813 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.91 KB/166819, heapSize ~190.09 KB/194648, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 948ms, sequenceid=311, compaction requested=false 2024-11-22T16:57:11,522 INFO [M:0;4618c6346924:38813 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T16:57:11,522 DEBUG [M:0;4618c6346924:38813 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732294630571Disabling compacts and flushes for region at 1732294630571Disabling writes for close at 1732294630572 (+1 ms)Obtaining lock to block concurrent updates at 1732294630572Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732294630572Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166819, getHeapSize=194888, getOffHeapSize=0, getCellsCount=358 at 1732294630572Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732294630573 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732294630573Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732294630591 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732294630591Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732294631008 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732294631034 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732294631034Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732294631451 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732294631478 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732294631478Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3663b23d: reopening flushed file at 1732294631493 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13346dcd: reopening flushed file at 1732294631502 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44693ff0: reopening flushed file at 1732294631509 (+7 ms)Finished flush of dataSize ~162.91 KB/166819, heapSize ~190.09 KB/194648, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 948ms, sequenceid=311, compaction requested=false at 1732294631520 (+11 ms)Writing region close event to WAL at 1732294631521 (+1 ms)Closed at 1732294631521 2024-11-22T16:57:11,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33941 is added to blk_1073741830_1006 (size=187641) 2024-11-22T16:57:11,528 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T16:57:11,528 INFO [M:0;4618c6346924:38813 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T16:57:11,528 INFO [M:0;4618c6346924:38813 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38813 2024-11-22T16:57:11,529 INFO [M:0;4618c6346924:38813 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T16:57:11,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:57:11,631 INFO [M:0;4618c6346924:38813 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T16:57:11,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38813-0x101409adc3d0000, quorum=127.0.0.1:57407, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T16:57:11,639 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-22T16:57:11,642 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T16:57:11,643 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T16:57:11,643 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T16:57:11,643 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/hadoop.log.dir/,STOPPED} 2024-11-22T16:57:11,646 WARN [BP-982762458-172.17.0.2-1732294535442 heartbeating to localhost/127.0.0.1:35361 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-22T16:57:11,646 WARN [BP-982762458-172.17.0.2-1732294535442 heartbeating to localhost/127.0.0.1:35361 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-982762458-172.17.0.2-1732294535442 (Datanode Uuid be375568-4cf4-4f76-bb45-a1c00c422d38) service to localhost/127.0.0.1:35361 2024-11-22T16:57:11,649 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-22T16:57:11,649 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-22T16:57:11,651 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/cluster_a6831cb5-71e9-f3e9-7e8c-861cbb1dc9c4/data/data1/current/BP-982762458-172.17.0.2-1732294535442 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T16:57:11,651 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/cluster_a6831cb5-71e9-f3e9-7e8c-861cbb1dc9c4/data/data2/current/BP-982762458-172.17.0.2-1732294535442 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-22T16:57:11,651 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-22T16:57:11,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-22T16:57:11,662 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-22T16:57:11,662 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-22T16:57:11,662 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-22T16:57:11,662 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d3811018-48f2-f62e-0898-127cda77c547/hadoop.log.dir/,STOPPED} 2024-11-22T16:57:11,672 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T16:57:11,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down