2024-12-03 16:08:22,712 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 16:08:22,724 main DEBUG Took 0.009949 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 16:08:22,724 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 16:08:22,725 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 16:08:22,726 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 16:08:22,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,743 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 16:08:22,759 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,760 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,761 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,761 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,761 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,762 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,762 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,763 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,763 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,763 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,764 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,765 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,765 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,766 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,766 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,767 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,767 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,768 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,768 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,768 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,769 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,769 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,769 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,770 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 16:08:22,770 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,770 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 16:08:22,772 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 16:08:22,774 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 16:08:22,777 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 16:08:22,777 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 16:08:22,779 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 16:08:22,779 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 16:08:22,791 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 16:08:22,795 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 16:08:22,797 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 16:08:22,798 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 16:08:22,798 main DEBUG createAppenders(={Console}) 2024-12-03 16:08:22,799 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-03 16:08:22,799 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 16:08:22,800 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-03 16:08:22,801 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 16:08:22,801 main DEBUG OutputStream closed 2024-12-03 16:08:22,801 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 16:08:22,802 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 16:08:22,802 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-03 16:08:22,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 16:08:22,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 16:08:22,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 16:08:22,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 16:08:22,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 16:08:22,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 16:08:22,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 16:08:22,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 16:08:22,893 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 16:08:22,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 16:08:22,894 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 16:08:22,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 16:08:22,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 16:08:22,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 16:08:22,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 16:08:22,896 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 16:08:22,897 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 16:08:22,898 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 16:08:22,900 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 16:08:22,900 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-03 16:08:22,900 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 16:08:22,901 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-03T16:08:22,920 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-03 16:08:22,924 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 16:08:22,924 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T16:08:23,217 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34 2024-12-03T16:08:23,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T16:08:23,257 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/cluster_2b1a8cb5-52d9-e7c9-2269-b219fee5ecf0, deleteOnExit=true 2024-12-03T16:08:23,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T16:08:23,259 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/test.cache.data in system properties and HBase conf 2024-12-03T16:08:23,260 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T16:08:23,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/hadoop.log.dir in system properties and HBase conf 2024-12-03T16:08:23,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T16:08:23,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T16:08:23,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T16:08:23,355 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T16:08:23,453 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T16:08:23,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T16:08:23,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T16:08:23,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T16:08:23,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T16:08:23,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T16:08:23,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T16:08:23,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T16:08:23,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T16:08:23,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T16:08:23,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/nfs.dump.dir in system properties and HBase conf 2024-12-03T16:08:23,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/java.io.tmpdir in system properties and HBase conf 2024-12-03T16:08:23,463 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T16:08:23,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T16:08:23,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T16:08:24,283 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T16:08:24,361 INFO [Time-limited test {}] log.Log(170): Logging initialized @2373ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T16:08:24,439 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T16:08:24,497 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T16:08:24,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T16:08:24,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T16:08:24,518 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T16:08:24,530 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T16:08:24,532 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/hadoop.log.dir/,AVAILABLE} 2024-12-03T16:08:24,533 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T16:08:24,726 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/java.io.tmpdir/jetty-localhost-42741-hadoop-hdfs-3_4_1-tests_jar-_-any-1563227435617618849/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T16:08:24,739 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:42741} 2024-12-03T16:08:24,739 INFO [Time-limited test {}] server.Server(415): Started @2752ms 2024-12-03T16:08:25,120 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T16:08:25,127 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T16:08:25,128 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T16:08:25,128 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T16:08:25,128 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T16:08:25,130 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/hadoop.log.dir/,AVAILABLE} 2024-12-03T16:08:25,130 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T16:08:25,256 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/java.io.tmpdir/jetty-localhost-44703-hadoop-hdfs-3_4_1-tests_jar-_-any-9209375289935859311/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T16:08:25,257 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:44703} 2024-12-03T16:08:25,257 INFO [Time-limited test {}] server.Server(415): Started @3270ms 2024-12-03T16:08:25,312 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T16:08:25,785 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/cluster_2b1a8cb5-52d9-e7c9-2269-b219fee5ecf0/data/data1/current/BP-473686499-172.17.0.3-1733242104029/current, will proceed with Du for space computation calculation, 2024-12-03T16:08:25,785 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/cluster_2b1a8cb5-52d9-e7c9-2269-b219fee5ecf0/data/data2/current/BP-473686499-172.17.0.3-1733242104029/current, will proceed with Du for space computation calculation, 2024-12-03T16:08:25,843 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T16:08:25,899 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb97582d6799d7f9b with lease ID 0x2153acfd14371865: Processing first storage report for DS-e1cf7398-9895-49d0-a74f-f55176a0183a from datanode DatanodeRegistration(127.0.0.1:45443, datanodeUuid=3822169c-e2f0-454f-bdc1-fd5bb72b2ba0, infoPort=38581, infoSecurePort=0, ipcPort=41659, storageInfo=lv=-57;cid=testClusterID;nsid=1661831339;c=1733242104029) 2024-12-03T16:08:25,901 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb97582d6799d7f9b with lease ID 0x2153acfd14371865: from storage DS-e1cf7398-9895-49d0-a74f-f55176a0183a node DatanodeRegistration(127.0.0.1:45443, datanodeUuid=3822169c-e2f0-454f-bdc1-fd5bb72b2ba0, infoPort=38581, infoSecurePort=0, ipcPort=41659, storageInfo=lv=-57;cid=testClusterID;nsid=1661831339;c=1733242104029), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T16:08:25,902 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb97582d6799d7f9b with lease ID 0x2153acfd14371865: Processing first storage report for DS-6f7acecd-b09c-40b8-8a41-1daac84dec30 from datanode DatanodeRegistration(127.0.0.1:45443, datanodeUuid=3822169c-e2f0-454f-bdc1-fd5bb72b2ba0, infoPort=38581, infoSecurePort=0, ipcPort=41659, storageInfo=lv=-57;cid=testClusterID;nsid=1661831339;c=1733242104029) 2024-12-03T16:08:25,902 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb97582d6799d7f9b with lease ID 0x2153acfd14371865: from storage DS-6f7acecd-b09c-40b8-8a41-1daac84dec30 node DatanodeRegistration(127.0.0.1:45443, datanodeUuid=3822169c-e2f0-454f-bdc1-fd5bb72b2ba0, infoPort=38581, infoSecurePort=0, ipcPort=41659, storageInfo=lv=-57;cid=testClusterID;nsid=1661831339;c=1733242104029), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T16:08:25,913 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34 2024-12-03T16:08:25,994 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/cluster_2b1a8cb5-52d9-e7c9-2269-b219fee5ecf0/zookeeper_0, clientPort=57365, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/cluster_2b1a8cb5-52d9-e7c9-2269-b219fee5ecf0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/cluster_2b1a8cb5-52d9-e7c9-2269-b219fee5ecf0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T16:08:26,004 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57365 2024-12-03T16:08:26,014 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T16:08:26,017 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T16:08:26,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741825_1001 (size=7) 2024-12-03T16:08:26,654 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c with version=8 2024-12-03T16:08:26,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/hbase-staging 2024-12-03T16:08:26,752 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T16:08:27,003 INFO [Time-limited test {}] client.ConnectionUtils(128): master/713339a81dd9:0 server-side Connection retries=9 2024-12-03T16:08:27,014 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T16:08:27,015 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T16:08:27,019 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T16:08:27,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T16:08:27,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T16:08:27,176 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T16:08:27,249 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T16:08:27,259 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T16:08:27,263 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T16:08:27,290 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8851 (auto-detected) 2024-12-03T16:08:27,292 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T16:08:27,312 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37115 2024-12-03T16:08:27,341 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37115 connecting to ZooKeeper ensemble=127.0.0.1:57365 2024-12-03T16:08:27,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:371150x0, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T16:08:27,378 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37115-0x101793116f70000 connected 2024-12-03T16:08:27,405 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T16:08:27,408 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T16:08:27,419 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T16:08:27,424 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c, hbase.cluster.distributed=false 2024-12-03T16:08:27,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T16:08:27,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37115 2024-12-03T16:08:27,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37115 2024-12-03T16:08:27,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37115 2024-12-03T16:08:27,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37115 2024-12-03T16:08:27,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37115 2024-12-03T16:08:27,579 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/713339a81dd9:0 server-side Connection retries=9 2024-12-03T16:08:27,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T16:08:27,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T16:08:27,581 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T16:08:27,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T16:08:27,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T16:08:27,585 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T16:08:27,587 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T16:08:27,588 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44209 2024-12-03T16:08:27,590 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44209 connecting to ZooKeeper ensemble=127.0.0.1:57365 2024-12-03T16:08:27,592 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T16:08:27,594 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T16:08:27,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442090x0, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T16:08:27,603 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44209-0x101793116f70001 connected 2024-12-03T16:08:27,603 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T16:08:27,608 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T16:08:27,622 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T16:08:27,625 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T16:08:27,630 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T16:08:27,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44209 2024-12-03T16:08:27,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44209 2024-12-03T16:08:27,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44209 2024-12-03T16:08:27,638 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44209 2024-12-03T16:08:27,641 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44209 2024-12-03T16:08:27,658 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;713339a81dd9:37115 2024-12-03T16:08:27,659 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/713339a81dd9,37115,1733242106809 2024-12-03T16:08:27,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T16:08:27,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T16:08:27,669 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/713339a81dd9,37115,1733242106809 2024-12-03T16:08:27,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:27,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T16:08:27,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:27,706 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T16:08:27,707 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/713339a81dd9,37115,1733242106809 from backup master directory 2024-12-03T16:08:27,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T16:08:27,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/713339a81dd9,37115,1733242106809 2024-12-03T16:08:27,712 WARN [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T16:08:27,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T16:08:27,712 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=713339a81dd9,37115,1733242106809 2024-12-03T16:08:27,715 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T16:08:27,716 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T16:08:27,782 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/hbase.id] with ID: 4205450e-d6c8-4f32-8dbe-0c92aac732ac 2024-12-03T16:08:27,783 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/.tmp/hbase.id 2024-12-03T16:08:27,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741826_1002 (size=42) 2024-12-03T16:08:28,198 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/.tmp/hbase.id]:[hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/hbase.id] 2024-12-03T16:08:28,250 INFO [master/713339a81dd9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T16:08:28,255 INFO [master/713339a81dd9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T16:08:28,277 INFO [master/713339a81dd9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-12-03T16:08:28,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:28,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:28,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741827_1003 (size=196) 2024-12-03T16:08:28,322 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:08:28,325 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T16:08:28,344 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T16:08:28,350 INFO [master/713339a81dd9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T16:08:28,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741828_1004 (size=1189) 2024-12-03T16:08:28,805 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store 2024-12-03T16:08:28,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741829_1005 (size=34) 2024-12-03T16:08:28,830 INFO [master/713339a81dd9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T16:08:28,834 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:08:28,835 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T16:08:28,835 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T16:08:28,835 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T16:08:28,837 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T16:08:28,837 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T16:08:28,837 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T16:08:28,838 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733242108835Disabling compacts and flushes for region at 1733242108835Disabling writes for close at 1733242108837 (+2 ms)Writing region close event to WAL at 1733242108837Closed at 1733242108837 2024-12-03T16:08:28,840 WARN [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/.initializing 2024-12-03T16:08:28,841 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/WALs/713339a81dd9,37115,1733242106809 2024-12-03T16:08:28,849 INFO [master/713339a81dd9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T16:08:28,865 INFO [master/713339a81dd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=713339a81dd9%2C37115%2C1733242106809, suffix=, logDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/WALs/713339a81dd9,37115,1733242106809, archiveDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/oldWALs, maxLogs=10 2024-12-03T16:08:28,889 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/WALs/713339a81dd9,37115,1733242106809/713339a81dd9%2C37115%2C1733242106809.1733242108870, exclude list is [], retry=0 2024-12-03T16:08:28,910 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45443,DS-e1cf7398-9895-49d0-a74f-f55176a0183a,DISK] 2024-12-03T16:08:28,914 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T16:08:28,959 INFO [master/713339a81dd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/WALs/713339a81dd9,37115,1733242106809/713339a81dd9%2C37115%2C1733242106809.1733242108870 2024-12-03T16:08:28,960 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38581:38581)] 2024-12-03T16:08:28,961 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:08:28,961 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:08:28,965 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:28,966 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,013 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T16:08:29,053 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:08:29,057 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T16:08:29,062 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:08:29,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T16:08:29,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:08:29,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T16:08:29,070 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:08:29,072 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,075 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,076 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,081 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,082 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,087 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T16:08:29,090 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T16:08:29,095 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:08:29,096 INFO [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73043427, jitterRate=0.08843187987804413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T16:08:29,103 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733242108979Initializing all the Stores at 1733242108981 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733242108982 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242108983 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242108983Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242108983Cleaning up temporary data from old regions at 1733242109082 (+99 ms)Region opened successfully at 1733242109103 (+21 ms) 2024-12-03T16:08:29,104 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T16:08:29,138 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74829c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=713339a81dd9/172.17.0.3:0 2024-12-03T16:08:29,172 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T16:08:29,183 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T16:08:29,183 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T16:08:29,186 INFO [master/713339a81dd9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T16:08:29,188 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T16:08:29,192 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T16:08:29,193 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T16:08:29,220 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T16:08:29,231 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T16:08:29,233 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T16:08:29,236 INFO [master/713339a81dd9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T16:08:29,238 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T16:08:29,239 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T16:08:29,242 INFO [master/713339a81dd9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T16:08:29,247 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T16:08:29,249 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T16:08:29,250 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T16:08:29,252 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T16:08:29,271 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T16:08:29,272 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T16:08:29,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T16:08:29,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T16:08:29,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:29,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:29,280 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=713339a81dd9,37115,1733242106809, sessionid=0x101793116f70000, setting cluster-up flag (Was=false) 2024-12-03T16:08:29,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:29,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:29,304 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T16:08:29,306 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=713339a81dd9,37115,1733242106809 2024-12-03T16:08:29,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:29,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:08:29,318 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T16:08:29,320 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=713339a81dd9,37115,1733242106809 2024-12-03T16:08:29,328 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T16:08:29,347 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(746): ClusterId : 4205450e-d6c8-4f32-8dbe-0c92aac732ac 2024-12-03T16:08:29,349 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T16:08:29,355 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T16:08:29,355 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T16:08:29,358 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T16:08:29,359 DEBUG [RS:0;713339a81dd9:44209 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a6d68d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=713339a81dd9/172.17.0.3:0 2024-12-03T16:08:29,376 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;713339a81dd9:44209 2024-12-03T16:08:29,380 INFO [RS:0;713339a81dd9:44209 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T16:08:29,380 INFO [RS:0;713339a81dd9:44209 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T16:08:29,381 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T16:08:29,384 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(2659): reportForDuty to master=713339a81dd9,37115,1733242106809 with port=44209, startcode=1733242107535 2024-12-03T16:08:29,398 DEBUG [RS:0;713339a81dd9:44209 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T16:08:29,408 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T16:08:29,419 INFO [master/713339a81dd9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T16:08:29,429 INFO [master/713339a81dd9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T16:08:29,435 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32823, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T16:08:29,438 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 713339a81dd9,37115,1733242106809 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T16:08:29,441 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37115 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T16:08:29,447 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/713339a81dd9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T16:08:29,448 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/713339a81dd9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T16:08:29,448 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/713339a81dd9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T16:08:29,448 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/713339a81dd9:0, corePoolSize=5, maxPoolSize=5 2024-12-03T16:08:29,448 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/713339a81dd9:0, corePoolSize=10, maxPoolSize=10 2024-12-03T16:08:29,449 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:08:29,449 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/713339a81dd9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T16:08:29,449 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:08:29,461 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733242139460 2024-12-03T16:08:29,463 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T16:08:29,464 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T16:08:29,464 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T16:08:29,464 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T16:08:29,469 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T16:08:29,469 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T16:08:29,470 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T16:08:29,470 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T16:08:29,471 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,471 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T16:08:29,471 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T16:08:29,474 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-03T16:08:29,474 WARN [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-12-03T16:08:29,475 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T16:08:29,476 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T16:08:29,476 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T16:08:29,478 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T16:08:29,479 INFO [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T16:08:29,483 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.large.0-1733242109480,5,FailOnTimeoutGroup] 2024-12-03T16:08:29,483 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.small.0-1733242109483,5,FailOnTimeoutGroup] 2024-12-03T16:08:29,483 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T16:08:29,484 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T16:08:29,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741831_1007 (size=1321) 2024-12-03T16:08:29,485 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T16:08:29,485 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T16:08:29,887 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T16:08:29,888 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:08:29,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741832_1008 (size=32) 2024-12-03T16:08:29,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:08:29,901 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T16:08:29,903 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T16:08:29,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,904 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:08:29,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T16:08:29,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T16:08:29,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:08:29,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T16:08:29,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T16:08:29,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:08:29,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T16:08:29,914 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T16:08:29,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:08:29,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:08:29,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T16:08:29,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740 2024-12-03T16:08:29,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740 2024-12-03T16:08:29,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T16:08:29,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T16:08:29,922 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T16:08:29,925 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T16:08:29,929 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:08:29,930 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68757667, jitterRate=0.024569079279899597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T16:08:29,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733242109899Initializing all the Stores at 1733242109900 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733242109901 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733242109901Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242109901Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733242109901Cleaning up temporary data from old regions at 1733242109921 (+20 ms)Region opened successfully at 1733242109934 (+13 ms) 2024-12-03T16:08:29,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T16:08:29,934 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T16:08:29,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T16:08:29,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T16:08:29,935 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T16:08:29,936 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T16:08:29,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733242109934Disabling compacts and flushes for region at 1733242109934Disabling writes for close at 1733242109935 (+1 ms)Writing region close event to WAL at 1733242109936 (+1 ms)Closed at 1733242109936 2024-12-03T16:08:29,940 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T16:08:29,941 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T16:08:29,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T16:08:29,956 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T16:08:29,959 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T16:08:30,110 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:31,111 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:32,112 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:33,113 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:34,118 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:35,124 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:36,131 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:37,142 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:37,249 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T16:08:37,250 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T16:08:38,194 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:39,198 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:40,205 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:41,213 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:42,262 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:43,281 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:44,282 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:45,284 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:46,290 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:47,297 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:48,304 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:49,317 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:50,318 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:51,322 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:52,324 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:53,335 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:54,335 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:55,338 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:55,910 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T16:08:56,339 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:57,340 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:58,344 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:59,346 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:08:59,491 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(2659): reportForDuty to master=713339a81dd9,37115,1733242106809 with port=44209, startcode=1733242107535 2024-12-03T16:08:59,527 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37115 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 713339a81dd9,44209,1733242107535 2024-12-03T16:08:59,574 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37115 {}] master.ServerManager(517): Registering regionserver=713339a81dd9,44209,1733242107535 2024-12-03T16:08:59,690 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:08:59,690 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41983 2024-12-03T16:08:59,690 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T16:08:59,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T16:08:59,710 DEBUG [RS:0;713339a81dd9:44209 {}] zookeeper.ZKUtil(111): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/713339a81dd9,44209,1733242107535 2024-12-03T16:08:59,711 WARN [RS:0;713339a81dd9:44209 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T16:08:59,711 INFO [RS:0;713339a81dd9:44209 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T16:08:59,711 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/WALs/713339a81dd9,44209,1733242107535 2024-12-03T16:08:59,808 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [713339a81dd9,44209,1733242107535] 2024-12-03T16:09:00,051 INFO [RS:0;713339a81dd9:44209 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T16:09:00,103 INFO [RS:0;713339a81dd9:44209 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T16:09:00,155 INFO [RS:0;713339a81dd9:44209 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T16:09:00,156 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,247 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T16:09:00,286 INFO [RS:0;713339a81dd9:44209 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T16:09:00,300 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,300 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,316 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,319 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,332 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,336 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,337 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/713339a81dd9:0, corePoolSize=2, maxPoolSize=2 2024-12-03T16:09:00,337 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,337 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,337 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,338 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,338 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,338 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/713339a81dd9:0, corePoolSize=1, maxPoolSize=1 2024-12-03T16:09:00,338 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/713339a81dd9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T16:09:00,338 DEBUG [RS:0;713339a81dd9:44209 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/713339a81dd9:0, corePoolSize=3, maxPoolSize=3 2024-12-03T16:09:00,353 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:00,468 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,469 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,469 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,469 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,469 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,469 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,44209,1733242107535-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T16:09:00,680 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T16:09:00,683 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,44209,1733242107535-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,722 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:00,723 INFO [RS:0;713339a81dd9:44209 {}] regionserver.Replication(171): 713339a81dd9,44209,1733242107535 started 2024-12-03T16:09:01,233 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:01,243 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1482): Serving as 713339a81dd9,44209,1733242107535, RpcServer on 713339a81dd9/172.17.0.3:44209, sessionid=0x101793116f70001 2024-12-03T16:09:01,244 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T16:09:01,245 DEBUG [RS:0;713339a81dd9:44209 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 713339a81dd9,44209,1733242107535 2024-12-03T16:09:01,245 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '713339a81dd9,44209,1733242107535' 2024-12-03T16:09:01,245 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T16:09:01,250 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T16:09:01,251 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T16:09:01,251 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T16:09:01,251 DEBUG [RS:0;713339a81dd9:44209 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 713339a81dd9,44209,1733242107535 2024-12-03T16:09:01,251 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '713339a81dd9,44209,1733242107535' 2024-12-03T16:09:01,251 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T16:09:01,270 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T16:09:01,271 DEBUG [RS:0;713339a81dd9:44209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T16:09:01,272 INFO [RS:0;713339a81dd9:44209 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T16:09:01,272 INFO [RS:0;713339a81dd9:44209 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T16:09:01,361 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:02,362 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:03,364 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:04,365 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:05,366 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:06,366 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:07,367 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:08,368 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:09,368 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:10,369 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:11,369 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:12,370 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:13,371 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:14,371 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:15,372 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:16,373 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:17,374 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:18,375 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:19,375 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:20,376 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:21,377 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:22,378 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:23,378 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:24,379 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:25,380 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:25,911 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T16:09:26,380 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:27,381 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:28,381 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:29,382 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:30,383 WARN [713339a81dd9:37115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T16:09:31,277 INFO [RS:0;713339a81dd9:44209 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T16:09:31,282 INFO [RS:0;713339a81dd9:44209 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=713339a81dd9%2C44209%2C1733242107535, suffix=, logDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/WALs/713339a81dd9,44209,1733242107535, archiveDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/oldWALs, maxLogs=32 2024-12-03T16:09:31,302 DEBUG [RS:0;713339a81dd9:44209 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/WALs/713339a81dd9,44209,1733242107535/713339a81dd9%2C44209%2C1733242107535.1733242171285, exclude list is [], retry=0 2024-12-03T16:09:31,307 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45443,DS-e1cf7398-9895-49d0-a74f-f55176a0183a,DISK] 2024-12-03T16:09:31,311 INFO [RS:0;713339a81dd9:44209 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/WALs/713339a81dd9,44209,1733242107535/713339a81dd9%2C44209%2C1733242107535.1733242171285 2024-12-03T16:09:31,312 DEBUG [RS:0;713339a81dd9:44209 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38581:38581)] 2024-12-03T16:09:31,385 DEBUG [713339a81dd9:37115 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T16:09:31,403 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:31,411 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 713339a81dd9,44209,1733242107535, state=OPENING 2024-12-03T16:09:31,417 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T16:09:31,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:09:31,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:09:31,421 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T16:09:31,421 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T16:09:31,422 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T16:09:31,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:09:31,604 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T16:09:31,608 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59759, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T16:09:31,620 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T16:09:31,620 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T16:09:31,621 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T16:09:31,624 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=713339a81dd9%2C44209%2C1733242107535.meta, suffix=.meta, logDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/WALs/713339a81dd9,44209,1733242107535, archiveDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/oldWALs, maxLogs=32 2024-12-03T16:09:31,644 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/WALs/713339a81dd9,44209,1733242107535/713339a81dd9%2C44209%2C1733242107535.meta.1733242171626.meta, exclude list is [], retry=0 2024-12-03T16:09:31,648 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45443,DS-e1cf7398-9895-49d0-a74f-f55176a0183a,DISK] 2024-12-03T16:09:31,651 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/WALs/713339a81dd9,44209,1733242107535/713339a81dd9%2C44209%2C1733242107535.meta.1733242171626.meta 2024-12-03T16:09:31,652 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38581:38581)] 2024-12-03T16:09:31,652 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:09:31,654 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T16:09:31,656 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T16:09:31,662 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T16:09:31,666 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T16:09:31,667 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:31,667 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T16:09:31,667 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T16:09:31,671 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T16:09:31,673 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T16:09:31,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:31,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:09:31,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T16:09:31,676 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T16:09:31,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:31,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:09:31,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T16:09:31,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T16:09:31,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:31,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:09:31,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T16:09:31,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T16:09:31,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:31,684 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T16:09:31,684 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T16:09:31,686 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740 2024-12-03T16:09:31,689 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740 2024-12-03T16:09:31,692 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T16:09:31,692 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T16:09:31,693 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T16:09:31,697 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T16:09:31,699 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63791764, jitterRate=-0.04942864179611206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T16:09:31,700 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T16:09:31,702 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733242171667Writing region info on filesystem at 1733242171668 (+1 ms)Initializing all the Stores at 1733242171670 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733242171670Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733242171670Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242171670Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733242171670Cleaning up temporary data from old regions at 1733242171692 (+22 ms)Running coprocessor post-open hooks at 1733242171700 (+8 ms)Region opened successfully at 1733242171702 (+2 ms) 2024-12-03T16:09:31,712 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733242171594 2024-12-03T16:09:31,728 DEBUG [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T16:09:31,729 INFO [RS_OPEN_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T16:09:31,731 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:31,734 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 713339a81dd9,44209,1733242107535, state=OPEN 2024-12-03T16:09:31,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T16:09:31,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T16:09:31,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T16:09:31,903 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T16:09:31,903 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=713339a81dd9,44209,1733242107535 2024-12-03T16:09:31,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T16:09:31,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=713339a81dd9,44209,1733242107535 in 478 msec 2024-12-03T16:09:31,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T16:09:31,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 1.965 sec 2024-12-03T16:09:31,920 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T16:09:31,920 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T16:09:31,939 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T16:09:31,941 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=713339a81dd9,44209,1733242107535, seqNum=-1] 2024-12-03T16:09:31,962 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T16:09:31,965 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41889, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T16:09:31,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 2.628 sec 2024-12-03T16:09:31,988 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733242171988, completionTime=-1 2024-12-03T16:09:31,990 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T16:09:31,990 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T16:09:32,019 INFO [master/713339a81dd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T16:09:32,019 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733242232019 2024-12-03T16:09:32,019 INFO [master/713339a81dd9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733242292019 2024-12-03T16:09:32,019 INFO [master/713339a81dd9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-12-03T16:09:32,021 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,37115,1733242106809-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:32,021 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,37115,1733242106809-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:32,021 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,37115,1733242106809-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:32,023 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-713339a81dd9:37115, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:32,023 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:32,024 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:32,031 DEBUG [master/713339a81dd9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T16:09:32,053 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 64.340sec 2024-12-03T16:09:32,054 INFO [master/713339a81dd9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T16:09:32,056 INFO [master/713339a81dd9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T16:09:32,057 INFO [master/713339a81dd9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T16:09:32,057 INFO [master/713339a81dd9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T16:09:32,058 INFO [master/713339a81dd9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T16:09:32,058 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,37115,1733242106809-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T16:09:32,059 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,37115,1733242106809-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T16:09:32,067 DEBUG [master/713339a81dd9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T16:09:32,068 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T16:09:32,069 INFO [master/713339a81dd9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=713339a81dd9,37115,1733242106809-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T16:09:32,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4225c540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T16:09:32,120 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T16:09:32,120 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T16:09:32,123 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 713339a81dd9,37115,-1 for getting cluster id 2024-12-03T16:09:32,126 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T16:09:32,135 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4205450e-d6c8-4f32-8dbe-0c92aac732ac' 2024-12-03T16:09:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T16:09:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4205450e-d6c8-4f32-8dbe-0c92aac732ac" 2024-12-03T16:09:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5bd34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T16:09:32,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [713339a81dd9,37115,-1] 2024-12-03T16:09:32,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T16:09:32,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:09:32,146 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T16:09:32,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@704e0d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T16:09:32,149 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T16:09:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=713339a81dd9,44209,1733242107535, seqNum=-1] 2024-12-03T16:09:32,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T16:09:32,160 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T16:09:32,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=713339a81dd9,37115,1733242106809 2024-12-03T16:09:32,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd93155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T16:09:32,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 713339a81dd9,37115,-1 for getting cluster id 2024-12-03T16:09:32,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T16:09:32,186 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4205450e-d6c8-4f32-8dbe-0c92aac732ac' 2024-12-03T16:09:32,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T16:09:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4205450e-d6c8-4f32-8dbe-0c92aac732ac" 2024-12-03T16:09:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8a2617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T16:09:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [713339a81dd9,37115,-1] 2024-12-03T16:09:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T16:09:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:09:32,191 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T16:09:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69e4f460, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T16:09:32,210 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=1379, ProcessCount=11, AvailableMemoryMB=1902 2024-12-03T16:09:32,217 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T16:09:32,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 713339a81dd9,37115,1733242106809 2024-12-03T16:09:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5fdd038e 2024-12-03T16:09:32,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T16:09:32,230 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T16:09:32,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:09:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T16:09:32,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:09:32,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-03T16:09:32,250 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:32,253 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:09:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T16:09:32,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741835_1011 (size=428) 2024-12-03T16:09:32,308 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e5df28ab31f4c59b2cf6c16f22d2fb13, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:09:32,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741836_1012 (size=79) 2024-12-03T16:09:32,726 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:32,726 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing e5df28ab31f4c59b2cf6c16f22d2fb13, disabling compactions & flushes 2024-12-03T16:09:32,726 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:09:32,726 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:09:32,726 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. after waiting 0 ms 2024-12-03T16:09:32,726 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:09:32,726 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:09:32,727 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e5df28ab31f4c59b2cf6c16f22d2fb13: Waiting for close lock at 1733242172726Disabling compacts and flushes for region at 1733242172726Disabling writes for close at 1733242172726Writing region close event to WAL at 1733242172726Closed at 1733242172726 2024-12-03T16:09:32,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:09:32,734 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733242172729"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242172729"}]},"ts":"1733242172729"} 2024-12-03T16:09:32,739 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:09:32,741 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:09:32,744 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242172741"}]},"ts":"1733242172741"} 2024-12-03T16:09:32,749 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-03T16:09:32,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e5df28ab31f4c59b2cf6c16f22d2fb13, ASSIGN}] 2024-12-03T16:09:32,753 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e5df28ab31f4c59b2cf6c16f22d2fb13, ASSIGN 2024-12-03T16:09:32,755 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e5df28ab31f4c59b2cf6c16f22d2fb13, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:09:32,906 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e5df28ab31f4c59b2cf6c16f22d2fb13, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:32,911 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e5df28ab31f4c59b2cf6c16f22d2fb13, ASSIGN because future has completed 2024-12-03T16:09:32,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5df28ab31f4c59b2cf6c16f22d2fb13, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:09:33,074 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:09:33,075 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e5df28ab31f4c59b2cf6c16f22d2fb13, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:09:33,075 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,075 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:33,076 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,076 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,078 INFO [StoreOpener-e5df28ab31f4c59b2cf6c16f22d2fb13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,080 INFO [StoreOpener-e5df28ab31f4c59b2cf6c16f22d2fb13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5df28ab31f4c59b2cf6c16f22d2fb13 columnFamilyName testFamily 2024-12-03T16:09:33,080 DEBUG [StoreOpener-e5df28ab31f4c59b2cf6c16f22d2fb13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:33,081 INFO [StoreOpener-e5df28ab31f4c59b2cf6c16f22d2fb13-1 {}] regionserver.HStore(327): Store=e5df28ab31f4c59b2cf6c16f22d2fb13/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:09:33,081 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,083 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,083 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,084 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,084 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,087 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,090 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:09:33,091 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e5df28ab31f4c59b2cf6c16f22d2fb13; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67774329, jitterRate=0.00991620123386383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:09:33,091 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:09:33,094 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e5df28ab31f4c59b2cf6c16f22d2fb13: Running coprocessor pre-open hook at 1733242173076Writing region info on filesystem at 1733242173076Initializing all the Stores at 1733242173077 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242173078 (+1 ms)Cleaning up temporary data from old regions at 1733242173084 (+6 ms)Running coprocessor post-open hooks at 1733242173091 (+7 ms)Region opened successfully at 1733242173094 (+3 ms) 2024-12-03T16:09:33,096 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13., pid=6, masterSystemTime=1733242173067 2024-12-03T16:09:33,100 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:09:33,100 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:09:33,101 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e5df28ab31f4c59b2cf6c16f22d2fb13, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:33,105 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e5df28ab31f4c59b2cf6c16f22d2fb13, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:09:33,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T16:09:33,111 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e5df28ab31f4c59b2cf6c16f22d2fb13, server=713339a81dd9,44209,1733242107535 in 195 msec 2024-12-03T16:09:33,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T16:09:33,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=e5df28ab31f4c59b2cf6c16f22d2fb13, ASSIGN in 360 msec 2024-12-03T16:09:33,116 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:09:33,117 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242173116"}]},"ts":"1733242173116"} 2024-12-03T16:09:33,120 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-03T16:09:33,121 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:09:33,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 882 msec 2024-12-03T16:09:33,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T16:09:33,306 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-03T16:09:33,306 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-03T16:09:33,307 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:33,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-03T16:09:33,314 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:33,314 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-03T16:09:33,317 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T16:09:33,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=713339a81dd9,44209,1733242107535, seqNum=-1] 2024-12-03T16:09:33,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T16:09:33,322 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T16:09:33,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:09:33,342 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:09:33,346 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-03T16:09:33,351 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T16:09:33,356 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T16:09:33,358 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T16:09:33,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8827874298434682872 2024-12-03T16:09:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:54124 deadline: 1733242176357, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T16:09:33,366 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T16:09:33,367 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T16:09:33,368 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:33,370 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:33,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8827874298434682870 2024-12-03T16:09:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:54124 deadline: 1733242176369, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T16:09:33,374 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:33,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-8827874298434682869 2024-12-03T16:09:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:54124 deadline: 1733242176374, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T16:09:33,379 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:33,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-8827874298434682868 2024-12-03T16:09:33,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:54124 deadline: 1733242176378, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T16:09:33,384 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:33,400 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-569714112_22 at /127.0.0.1:47874 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/713339a81dd9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1316 (was 1379), ProcessCount=11 (was 11), AvailableMemoryMB=1850 (was 1902) 2024-12-03T16:09:33,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1316, ProcessCount=11, AvailableMemoryMB=1849 2024-12-03T16:09:33,416 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:09:33,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-03T16:09:33,420 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:09:33,421 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:33,421 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-03T16:09:33,422 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:09:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T16:09:33,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741837_1013 (size=414) 2024-12-03T16:09:33,836 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3db5b4cbdc1bdb1a2d5113b613ab8786, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:09:33,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741838_1014 (size=65) 2024-12-03T16:09:34,247 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:34,248 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 3db5b4cbdc1bdb1a2d5113b613ab8786, disabling compactions & flushes 2024-12-03T16:09:34,248 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:09:34,248 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:09:34,248 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. after waiting 0 ms 2024-12-03T16:09:34,248 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:09:34,248 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:09:34,248 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3db5b4cbdc1bdb1a2d5113b613ab8786: Waiting for close lock at 1733242174247Disabling compacts and flushes for region at 1733242174247Disabling writes for close at 1733242174248 (+1 ms)Writing region close event to WAL at 1733242174248Closed at 1733242174248 2024-12-03T16:09:34,250 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:09:34,250 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733242174250"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242174250"}]},"ts":"1733242174250"} 2024-12-03T16:09:34,254 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:09:34,256 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:09:34,256 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242174256"}]},"ts":"1733242174256"} 2024-12-03T16:09:34,259 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T16:09:34,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3db5b4cbdc1bdb1a2d5113b613ab8786, ASSIGN}] 2024-12-03T16:09:34,261 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3db5b4cbdc1bdb1a2d5113b613ab8786, ASSIGN 2024-12-03T16:09:34,263 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3db5b4cbdc1bdb1a2d5113b613ab8786, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:09:34,414 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=3db5b4cbdc1bdb1a2d5113b613ab8786, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:34,419 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3db5b4cbdc1bdb1a2d5113b613ab8786, ASSIGN because future has completed 2024-12-03T16:09:34,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3db5b4cbdc1bdb1a2d5113b613ab8786, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:09:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T16:09:34,581 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:09:34,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3db5b4cbdc1bdb1a2d5113b613ab8786, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:09:34,582 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,582 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:34,582 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,582 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,584 INFO [StoreOpener-3db5b4cbdc1bdb1a2d5113b613ab8786-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,587 INFO [StoreOpener-3db5b4cbdc1bdb1a2d5113b613ab8786-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3db5b4cbdc1bdb1a2d5113b613ab8786 columnFamilyName testFamily 2024-12-03T16:09:34,587 DEBUG [StoreOpener-3db5b4cbdc1bdb1a2d5113b613ab8786-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:34,588 INFO [StoreOpener-3db5b4cbdc1bdb1a2d5113b613ab8786-1 {}] regionserver.HStore(327): Store=3db5b4cbdc1bdb1a2d5113b613ab8786/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:09:34,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,590 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,591 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,591 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,597 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:09:34,599 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 3db5b4cbdc1bdb1a2d5113b613ab8786; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71604363, jitterRate=0.06698815524578094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:09:34,599 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:09:34,600 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3db5b4cbdc1bdb1a2d5113b613ab8786: Running coprocessor pre-open hook at 1733242174582Writing region info on filesystem at 1733242174582Initializing all the Stores at 1733242174584 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242174584Cleaning up temporary data from old regions at 1733242174591 (+7 ms)Running coprocessor post-open hooks at 1733242174599 (+8 ms)Region opened successfully at 1733242174599 2024-12-03T16:09:34,601 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786., pid=9, masterSystemTime=1733242174574 2024-12-03T16:09:34,605 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:09:34,605 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:09:34,607 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=3db5b4cbdc1bdb1a2d5113b613ab8786, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:34,610 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3db5b4cbdc1bdb1a2d5113b613ab8786, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:09:34,617 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-03T16:09:34,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 3db5b4cbdc1bdb1a2d5113b613ab8786, server=713339a81dd9,44209,1733242107535 in 192 msec 2024-12-03T16:09:34,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T16:09:34,621 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=3db5b4cbdc1bdb1a2d5113b613ab8786, ASSIGN in 358 msec 2024-12-03T16:09:34,622 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:09:34,623 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242174622"}]},"ts":"1733242174622"} 2024-12-03T16:09:34,626 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T16:09:34,628 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:09:34,632 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2120 sec 2024-12-03T16:09:35,882 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-03T16:09:35,883 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T16:09:35,884 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-03T16:09:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T16:09:36,445 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T16:09:36,445 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-03T16:09:36,446 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:36,451 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T16:09:36,452 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:36,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-03T16:09:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:09:36,472 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:09:36,472 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T16:09:36,477 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T16:09:36,479 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T16:09:37,205 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:09:37,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T16:09:37,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T16:09:37,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T16:09:37,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T16:09:37,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T16:09:37,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T16:09:37,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T16:09:37,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T16:09:42,488 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786., hostname=713339a81dd9,44209,1733242107535, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786., hostname=713339a81dd9,44209,1733242107535, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=713339a81dd9:44209 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-03T16:09:42,488 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786., hostname=713339a81dd9,44209,1733242107535, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-12-03T16:09:42,488 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786., hostname=713339a81dd9,44209,1733242107535, seqNum=2 from cache 2024-12-03T16:09:42,501 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1137 (was 1316), ProcessCount=11 (was 11), AvailableMemoryMB=1610 (was 1849) 2024-12-03T16:09:42,510 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1137, ProcessCount=11, AvailableMemoryMB=1610 2024-12-03T16:09:42,513 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:09:42,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-03T16:09:42,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:09:42,517 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:42,517 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-03T16:09:42,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T16:09:42,519 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:09:42,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741839_1015 (size=412) 2024-12-03T16:09:42,931 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 482f4868465500eb74348718fb936e79, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:09:42,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741840_1016 (size=63) 2024-12-03T16:09:42,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-12-03T16:09:43,345 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:43,345 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 482f4868465500eb74348718fb936e79, disabling compactions & flushes 2024-12-03T16:09:43,345 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:09:43,345 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:09:43,345 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. after waiting 0 ms 2024-12-03T16:09:43,345 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:09:43,346 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:09:43,346 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 482f4868465500eb74348718fb936e79: Waiting for close lock at 1733242183345Disabling compacts and flushes for region at 1733242183345Disabling writes for close at 1733242183345Writing region close event to WAL at 1733242183346 (+1 ms)Closed at 1733242183346 2024-12-03T16:09:43,348 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:09:43,348 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733242183348"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242183348"}]},"ts":"1733242183348"} 2024-12-03T16:09:43,351 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:09:43,353 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:09:43,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242183353"}]},"ts":"1733242183353"} 2024-12-03T16:09:43,356 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T16:09:43,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=482f4868465500eb74348718fb936e79, ASSIGN}] 2024-12-03T16:09:43,358 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=482f4868465500eb74348718fb936e79, ASSIGN 2024-12-03T16:09:43,360 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=482f4868465500eb74348718fb936e79, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:09:43,511 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=482f4868465500eb74348718fb936e79, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:43,515 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=482f4868465500eb74348718fb936e79, ASSIGN because future has completed 2024-12-03T16:09:43,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 482f4868465500eb74348718fb936e79, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:09:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T16:09:43,675 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:09:43,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 482f4868465500eb74348718fb936e79, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:09:43,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:43,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,678 INFO [StoreOpener-482f4868465500eb74348718fb936e79-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,680 INFO [StoreOpener-482f4868465500eb74348718fb936e79-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 482f4868465500eb74348718fb936e79 columnFamilyName testFamily 2024-12-03T16:09:43,680 DEBUG [StoreOpener-482f4868465500eb74348718fb936e79-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:43,681 INFO [StoreOpener-482f4868465500eb74348718fb936e79-1 {}] regionserver.HStore(327): Store=482f4868465500eb74348718fb936e79/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:09:43,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,682 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,683 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,683 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,683 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,686 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,689 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:09:43,690 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 482f4868465500eb74348718fb936e79; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69072616, jitterRate=0.029262185096740723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:09:43,690 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 482f4868465500eb74348718fb936e79 2024-12-03T16:09:43,691 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 482f4868465500eb74348718fb936e79: Running coprocessor pre-open hook at 1733242183676Writing region info on filesystem at 1733242183676Initializing all the Stores at 1733242183677 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242183678 (+1 ms)Cleaning up temporary data from old regions at 1733242183683 (+5 ms)Running coprocessor post-open hooks at 1733242183690 (+7 ms)Region opened successfully at 1733242183691 (+1 ms) 2024-12-03T16:09:43,692 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79., pid=12, masterSystemTime=1733242183669 2024-12-03T16:09:43,695 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:09:43,695 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:09:43,696 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=482f4868465500eb74348718fb936e79, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:43,700 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 482f4868465500eb74348718fb936e79, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:09:43,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T16:09:43,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 482f4868465500eb74348718fb936e79, server=713339a81dd9,44209,1733242107535 in 186 msec 2024-12-03T16:09:43,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T16:09:43,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=482f4868465500eb74348718fb936e79, ASSIGN in 348 msec 2024-12-03T16:09:43,708 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:09:43,709 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242183708"}]},"ts":"1733242183708"} 2024-12-03T16:09:43,711 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T16:09:43,712 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:09:43,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.1990 sec 2024-12-03T16:09:45,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T16:09:45,555 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-03T16:09:45,555 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T16:09:45,555 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:45,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T16:09:45,560 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:45,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-03T16:09:45,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:09:45,577 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:09:45,577 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T16:09:45,581 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T16:09:45,584 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T16:09:45,586 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T16:09:45,587 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T16:09:47,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T16:09:47,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T16:09:47,886 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-03T16:09:49,203 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:09:51,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682859' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T16:09:51,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44209 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:54124 deadline: 1733242197592, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682859' does not exist 2024-12-03T16:09:51,599 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 713339a81dd9,44209,1733242107535 for closing scanner id = -8827874298434682859 for 482f4868465500eb74348718fb936e79 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682859' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T16:09:51,603 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1062 (was 1137), ProcessCount=11 (was 11), AvailableMemoryMB=1445 (was 1610) 2024-12-03T16:09:51,614 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=1062, ProcessCount=11, AvailableMemoryMB=1444 2024-12-03T16:09:51,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:09:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-03T16:09:51,621 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:09:51,621 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:51,621 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-03T16:09:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T16:09:51,623 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:09:51,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741841_1017 (size=423) 2024-12-03T16:09:52,035 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 847db7889994e22b30682d85199e889c, NAME => 'testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:09:52,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741842_1018 (size=74) 2024-12-03T16:09:52,044 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:52,045 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 847db7889994e22b30682d85199e889c, disabling compactions & flushes 2024-12-03T16:09:52,045 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:09:52,045 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:09:52,045 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. after waiting 0 ms 2024-12-03T16:09:52,045 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:09:52,045 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:09:52,045 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 847db7889994e22b30682d85199e889c: Waiting for close lock at 1733242192045Disabling compacts and flushes for region at 1733242192045Disabling writes for close at 1733242192045Writing region close event to WAL at 1733242192045Closed at 1733242192045 2024-12-03T16:09:52,047 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:09:52,048 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733242192047"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242192047"}]},"ts":"1733242192047"} 2024-12-03T16:09:52,051 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:09:52,053 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:09:52,053 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242192053"}]},"ts":"1733242192053"} 2024-12-03T16:09:52,057 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-03T16:09:52,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=847db7889994e22b30682d85199e889c, ASSIGN}] 2024-12-03T16:09:52,059 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=847db7889994e22b30682d85199e889c, ASSIGN 2024-12-03T16:09:52,060 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=847db7889994e22b30682d85199e889c, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:09:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-12-03T16:09:52,211 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=847db7889994e22b30682d85199e889c, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:52,215 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=847db7889994e22b30682d85199e889c, ASSIGN because future has completed 2024-12-03T16:09:52,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 847db7889994e22b30682d85199e889c, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:09:52,375 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:09:52,376 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 847db7889994e22b30682d85199e889c, NAME => 'testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:09:52,376 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,376 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:52,377 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,377 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,379 INFO [StoreOpener-847db7889994e22b30682d85199e889c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,382 INFO [StoreOpener-847db7889994e22b30682d85199e889c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 847db7889994e22b30682d85199e889c columnFamilyName testFamily 2024-12-03T16:09:52,382 DEBUG [StoreOpener-847db7889994e22b30682d85199e889c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:52,383 INFO [StoreOpener-847db7889994e22b30682d85199e889c-1 {}] regionserver.HStore(327): Store=847db7889994e22b30682d85199e889c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:09:52,383 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,385 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,386 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,386 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,386 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,389 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,393 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:09:52,394 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 847db7889994e22b30682d85199e889c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70323341, jitterRate=0.04789943993091583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:09:52,395 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 847db7889994e22b30682d85199e889c 2024-12-03T16:09:52,395 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 847db7889994e22b30682d85199e889c: Running coprocessor pre-open hook at 1733242192377Writing region info on filesystem at 1733242192377Initializing all the Stores at 1733242192378 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242192379 (+1 ms)Cleaning up temporary data from old regions at 1733242192386 (+7 ms)Running coprocessor post-open hooks at 1733242192395 (+9 ms)Region opened successfully at 1733242192395 2024-12-03T16:09:52,397 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c., pid=15, masterSystemTime=1733242192369 2024-12-03T16:09:52,400 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:09:52,401 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:09:52,402 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=847db7889994e22b30682d85199e889c, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:52,405 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 847db7889994e22b30682d85199e889c, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:09:52,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T16:09:52,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 847db7889994e22b30682d85199e889c, server=713339a81dd9,44209,1733242107535 in 192 msec 2024-12-03T16:09:52,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T16:09:52,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=847db7889994e22b30682d85199e889c, ASSIGN in 355 msec 2024-12-03T16:09:52,418 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:09:52,418 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242192418"}]},"ts":"1733242192418"} 2024-12-03T16:09:52,421 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-03T16:09:52,423 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:09:52,426 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 806 msec 2024-12-03T16:09:52,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T16:09:52,635 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-03T16:09:52,635 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-03T16:09:52,636 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:52,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-03T16:09:52,641 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:52,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-03T16:09:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:09:52,659 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:09:52,659 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-03T16:09:52,663 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T16:09:52,666 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T16:09:52,667 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T16:09:52,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8827874298434682855 2024-12-03T16:09:52,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:54124 deadline: 1733242195667, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T16:09:52,671 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T16:09:52,672 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T16:09:52,673 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:52,674 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:52,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8827874298434682853 2024-12-03T16:09:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:54124 deadline: 1733242195674, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T16:09:52,679 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:52,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-8827874298434682852 2024-12-03T16:09:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:54124 deadline: 1733242195678, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T16:09:52,682 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:52,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-8827874298434682851 2024-12-03T16:09:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:54124 deadline: 1733242195681, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T16:09:52,685 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T16:09:52,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-569714112_22 at /127.0.0.1:46812 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=985 (was 1062), ProcessCount=11 (was 11), AvailableMemoryMB=1438 (was 1444) 2024-12-03T16:09:52,707 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=985, ProcessCount=11, AvailableMemoryMB=1438 2024-12-03T16:09:52,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:09:52,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-03T16:09:52,713 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:09:52,713 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:52,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-03T16:09:52,715 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:09:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T16:09:52,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741843_1019 (size=418) 2024-12-03T16:09:53,127 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 475f427f43d48197a370cc1148063e41, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:09:53,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741844_1020 (size=69) 2024-12-03T16:09:53,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:53,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 475f427f43d48197a370cc1148063e41, disabling compactions & flushes 2024-12-03T16:09:53,538 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:09:53,538 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:09:53,539 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. after waiting 1 ms 2024-12-03T16:09:53,539 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:09:53,539 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:09:53,539 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 475f427f43d48197a370cc1148063e41: Waiting for close lock at 1733242193538Disabling compacts and flushes for region at 1733242193538Disabling writes for close at 1733242193539 (+1 ms)Writing region close event to WAL at 1733242193539Closed at 1733242193539 2024-12-03T16:09:53,541 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:09:53,541 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733242193541"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242193541"}]},"ts":"1733242193541"} 2024-12-03T16:09:53,545 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:09:53,547 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:09:53,547 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242193547"}]},"ts":"1733242193547"} 2024-12-03T16:09:53,550 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T16:09:53,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=475f427f43d48197a370cc1148063e41, ASSIGN}] 2024-12-03T16:09:53,552 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=475f427f43d48197a370cc1148063e41, ASSIGN 2024-12-03T16:09:53,553 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=475f427f43d48197a370cc1148063e41, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:09:53,704 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=475f427f43d48197a370cc1148063e41, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:53,708 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=475f427f43d48197a370cc1148063e41, ASSIGN because future has completed 2024-12-03T16:09:53,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 475f427f43d48197a370cc1148063e41, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:09:53,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T16:09:53,868 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:09:53,868 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 475f427f43d48197a370cc1148063e41, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:09:53,869 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,869 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:53,869 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,869 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,871 INFO [StoreOpener-475f427f43d48197a370cc1148063e41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,873 INFO [StoreOpener-475f427f43d48197a370cc1148063e41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 475f427f43d48197a370cc1148063e41 columnFamilyName testFamily 2024-12-03T16:09:53,874 DEBUG [StoreOpener-475f427f43d48197a370cc1148063e41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:53,874 INFO [StoreOpener-475f427f43d48197a370cc1148063e41-1 {}] regionserver.HStore(327): Store=475f427f43d48197a370cc1148063e41/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:09:53,875 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,876 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,877 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,877 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,877 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,880 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,883 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:09:53,884 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 475f427f43d48197a370cc1148063e41; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66953664, jitterRate=-0.0023126602172851562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:09:53,884 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 475f427f43d48197a370cc1148063e41 2024-12-03T16:09:53,884 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 475f427f43d48197a370cc1148063e41: Running coprocessor pre-open hook at 1733242193869Writing region info on filesystem at 1733242193869Initializing all the Stores at 1733242193871 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242193871Cleaning up temporary data from old regions at 1733242193877 (+6 ms)Running coprocessor post-open hooks at 1733242193884 (+7 ms)Region opened successfully at 1733242193884 2024-12-03T16:09:53,886 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., pid=18, masterSystemTime=1733242193862 2024-12-03T16:09:53,887 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-03T16:09:53,889 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:09:53,889 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:09:53,890 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=475f427f43d48197a370cc1148063e41, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:53,893 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 475f427f43d48197a370cc1148063e41, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:09:53,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T16:09:53,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 475f427f43d48197a370cc1148063e41, server=713339a81dd9,44209,1733242107535 in 186 msec 2024-12-03T16:09:53,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T16:09:53,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=475f427f43d48197a370cc1148063e41, ASSIGN in 348 msec 2024-12-03T16:09:53,902 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:09:53,902 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242193902"}]},"ts":"1733242193902"} 2024-12-03T16:09:53,905 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T16:09:53,906 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:09:53,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1970 sec 2024-12-03T16:09:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T16:09:55,735 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-03T16:09:55,735 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T16:09:55,735 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:55,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T16:09:55,740 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:09:55,740 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-03T16:09:55,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:09:55,757 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:09:55,757 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T16:09:55,761 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T16:09:55,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T16:09:55,911 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T16:09:56,771 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=713339a81dd9:44209 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-03T16:09:56,771 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1009ms, rpcTimeout=1000ms 2024-12-03T16:09:56,772 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2 from cache 2024-12-03T16:09:57,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T16:09:57,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-03T16:09:57,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T16:09:57,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T16:09:57,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-12-03T16:09:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:09:57,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T16:09:57,879 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:09:58,771 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=713339a81dd9:44209 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=973ms 2024-12-03T16:09:58,771 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=973ms 2024-12-03T16:09:58,771 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., hostname=713339a81dd9,44209,1733242107535, seqNum=2 from cache 2024-12-03T16:09:58,781 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=930 (was 985), ProcessCount=11 (was 11), AvailableMemoryMB=1422 (was 1438) 2024-12-03T16:09:58,789 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=930, ProcessCount=11, AvailableMemoryMB=1422 2024-12-03T16:09:58,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:09:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-03T16:09:58,795 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:09:58,796 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:58,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-03T16:09:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T16:09:58,797 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:09:58,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741845_1021 (size=416) 2024-12-03T16:09:59,210 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dc0fbe6bdf3499b5dceba341c33d4289, NAME => 'testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:09:59,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741846_1022 (size=67) 2024-12-03T16:09:59,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-12-03T16:09:59,621 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:59,621 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing dc0fbe6bdf3499b5dceba341c33d4289, disabling compactions & flushes 2024-12-03T16:09:59,621 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:09:59,622 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:09:59,622 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. after waiting 0 ms 2024-12-03T16:09:59,622 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:09:59,622 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:09:59,622 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for dc0fbe6bdf3499b5dceba341c33d4289: Waiting for close lock at 1733242199621Disabling compacts and flushes for region at 1733242199621Disabling writes for close at 1733242199622 (+1 ms)Writing region close event to WAL at 1733242199622Closed at 1733242199622 2024-12-03T16:09:59,625 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:09:59,625 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733242199625"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242199625"}]},"ts":"1733242199625"} 2024-12-03T16:09:59,628 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:09:59,630 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:09:59,630 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242199630"}]},"ts":"1733242199630"} 2024-12-03T16:09:59,633 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T16:09:59,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dc0fbe6bdf3499b5dceba341c33d4289, ASSIGN}] 2024-12-03T16:09:59,634 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dc0fbe6bdf3499b5dceba341c33d4289, ASSIGN 2024-12-03T16:09:59,636 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dc0fbe6bdf3499b5dceba341c33d4289, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:09:59,786 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=dc0fbe6bdf3499b5dceba341c33d4289, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:59,790 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dc0fbe6bdf3499b5dceba341c33d4289, ASSIGN because future has completed 2024-12-03T16:09:59,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc0fbe6bdf3499b5dceba341c33d4289, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:09:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T16:09:59,888 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-03T16:09:59,949 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:09:59,949 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => dc0fbe6bdf3499b5dceba341c33d4289, NAME => 'testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:09:59,950 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,950 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:09:59,951 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,951 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,953 INFO [StoreOpener-dc0fbe6bdf3499b5dceba341c33d4289-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,955 INFO [StoreOpener-dc0fbe6bdf3499b5dceba341c33d4289-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc0fbe6bdf3499b5dceba341c33d4289 columnFamilyName testFamily 2024-12-03T16:09:59,955 DEBUG [StoreOpener-dc0fbe6bdf3499b5dceba341c33d4289-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:09:59,955 INFO [StoreOpener-dc0fbe6bdf3499b5dceba341c33d4289-1 {}] regionserver.HStore(327): Store=dc0fbe6bdf3499b5dceba341c33d4289/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:09:59,956 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,957 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,957 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,958 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,958 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,961 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,964 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:09:59,965 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened dc0fbe6bdf3499b5dceba341c33d4289; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71378148, jitterRate=0.0636172890663147}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:09:59,965 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:09:59,966 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for dc0fbe6bdf3499b5dceba341c33d4289: Running coprocessor pre-open hook at 1733242199951Writing region info on filesystem at 1733242199951Initializing all the Stores at 1733242199952 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242199952Cleaning up temporary data from old regions at 1733242199958 (+6 ms)Running coprocessor post-open hooks at 1733242199965 (+7 ms)Region opened successfully at 1733242199966 (+1 ms) 2024-12-03T16:09:59,967 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289., pid=21, masterSystemTime=1733242199943 2024-12-03T16:09:59,970 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:09:59,970 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:09:59,971 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=dc0fbe6bdf3499b5dceba341c33d4289, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:09:59,975 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc0fbe6bdf3499b5dceba341c33d4289, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:09:59,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-03T16:09:59,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure dc0fbe6bdf3499b5dceba341c33d4289, server=713339a81dd9,44209,1733242107535 in 186 msec 2024-12-03T16:09:59,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T16:09:59,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=dc0fbe6bdf3499b5dceba341c33d4289, ASSIGN in 347 msec 2024-12-03T16:09:59,984 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:09:59,985 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242199984"}]},"ts":"1733242199984"} 2024-12-03T16:09:59,988 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T16:09:59,989 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:09:59,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.1970 sec 2024-12-03T16:10:01,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T16:10:01,815 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-03T16:10:01,815 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T16:10:01,815 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:01,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T16:10:01,820 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:01,820 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-03T16:10:01,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:10:01,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:10:01,838 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T16:10:01,842 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T16:10:01,844 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T16:10:01,846 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T16:10:01,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T16:10:04,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682840' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T16:10:04,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44209 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:54124 deadline: 1733242205852, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682840' does not exist 2024-12-03T16:10:04,855 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 713339a81dd9,44209,1733242107535 for closing scanner id = -8827874298434682840 for dc0fbe6bdf3499b5dceba341c33d4289 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682840' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T16:10:04,861 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=887 (was 930), ProcessCount=11 (was 11), AvailableMemoryMB=1385 (was 1422) 2024-12-03T16:10:04,870 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=887, ProcessCount=11, AvailableMemoryMB=1384 2024-12-03T16:10:04,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:10:04,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-03T16:10:04,876 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:10:04,876 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:10:04,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-03T16:10:04,878 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:10:04,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T16:10:04,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741847_1023 (size=411) 2024-12-03T16:10:05,291 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 92c6012d33edc09803453b857b83ee01, NAME => 'testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:10:05,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741848_1024 (size=62) 2024-12-03T16:10:05,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-03T16:10:05,452 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:10:05,702 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:10:05,702 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 92c6012d33edc09803453b857b83ee01, disabling compactions & flushes 2024-12-03T16:10:05,702 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:05,702 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:05,702 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. after waiting 0 ms 2024-12-03T16:10:05,702 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:05,702 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:05,702 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 92c6012d33edc09803453b857b83ee01: Waiting for close lock at 1733242205702Disabling compacts and flushes for region at 1733242205702Disabling writes for close at 1733242205702Writing region close event to WAL at 1733242205702Closed at 1733242205702 2024-12-03T16:10:05,704 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:10:05,704 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733242205704"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242205704"}]},"ts":"1733242205704"} 2024-12-03T16:10:05,707 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:10:05,708 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:10:05,709 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242205708"}]},"ts":"1733242205708"} 2024-12-03T16:10:05,711 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T16:10:05,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=92c6012d33edc09803453b857b83ee01, ASSIGN}] 2024-12-03T16:10:05,713 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=92c6012d33edc09803453b857b83ee01, ASSIGN 2024-12-03T16:10:05,714 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=92c6012d33edc09803453b857b83ee01, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:10:05,865 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=92c6012d33edc09803453b857b83ee01, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:10:05,869 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=92c6012d33edc09803453b857b83ee01, ASSIGN because future has completed 2024-12-03T16:10:05,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92c6012d33edc09803453b857b83ee01, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:10:05,889 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-03T16:10:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T16:10:06,028 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:06,028 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 92c6012d33edc09803453b857b83ee01, NAME => 'testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:10:06,028 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,029 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:10:06,029 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,029 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,031 INFO [StoreOpener-92c6012d33edc09803453b857b83ee01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,033 INFO [StoreOpener-92c6012d33edc09803453b857b83ee01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92c6012d33edc09803453b857b83ee01 columnFamilyName testFamily 2024-12-03T16:10:06,033 DEBUG [StoreOpener-92c6012d33edc09803453b857b83ee01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:10:06,034 INFO [StoreOpener-92c6012d33edc09803453b857b83ee01-1 {}] regionserver.HStore(327): Store=92c6012d33edc09803453b857b83ee01/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:10:06,034 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,035 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,036 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,037 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,037 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,039 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,042 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:10:06,043 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 92c6012d33edc09803453b857b83ee01; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69030595, jitterRate=0.02863602340221405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:10:06,043 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:06,044 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 92c6012d33edc09803453b857b83ee01: Running coprocessor pre-open hook at 1733242206029Writing region info on filesystem at 1733242206029Initializing all the Stores at 1733242206030 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242206030Cleaning up temporary data from old regions at 1733242206037 (+7 ms)Running coprocessor post-open hooks at 1733242206043 (+6 ms)Region opened successfully at 1733242206043 2024-12-03T16:10:06,045 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01., pid=24, masterSystemTime=1733242206022 2024-12-03T16:10:06,048 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:06,048 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:06,049 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=92c6012d33edc09803453b857b83ee01, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:10:06,052 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92c6012d33edc09803453b857b83ee01, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:10:06,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T16:10:06,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 92c6012d33edc09803453b857b83ee01, server=713339a81dd9,44209,1733242107535 in 184 msec 2024-12-03T16:10:06,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T16:10:06,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=92c6012d33edc09803453b857b83ee01, ASSIGN in 346 msec 2024-12-03T16:10:06,061 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:10:06,061 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242206061"}]},"ts":"1733242206061"} 2024-12-03T16:10:06,064 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T16:10:06,066 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:10:06,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.1930 sec 2024-12-03T16:10:07,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T16:10:07,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T16:10:07,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T16:10:07,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T16:10:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T16:10:07,915 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-03T16:10:07,915 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T16:10:07,915 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:07,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T16:10:07,920 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:07,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-03T16:10:07,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:10:07,935 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:10:07,935 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T16:10:07,938 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T16:10:07,939 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T16:10:07,941 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T16:10:07,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T16:10:10,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682836' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T16:10:10,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44209 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:54124 deadline: 1733242211944, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682836' does not exist 2024-12-03T16:10:10,945 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 713339a81dd9,44209,1733242107535 for closing scanner id = -8827874298434682836 for 92c6012d33edc09803453b857b83ee01 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682836' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T16:10:10,951 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=848 (was 887), ProcessCount=11 (was 11), AvailableMemoryMB=1359 (was 1384) 2024-12-03T16:10:10,959 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=848, ProcessCount=11, AvailableMemoryMB=1359 2024-12-03T16:10:10,962 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:10:10,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T16:10:10,965 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:10:10,965 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:10:10,965 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-03T16:10:10,966 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:10:10,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T16:10:10,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741849_1025 (size=419) 2024-12-03T16:10:11,377 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cce9b46f903d84794859c85b38505545, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:10:11,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741850_1026 (size=70) 2024-12-03T16:10:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-12-03T16:10:11,530 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:10:11,788 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:10:11,789 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing cce9b46f903d84794859c85b38505545, disabling compactions & flushes 2024-12-03T16:10:11,789 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:11,789 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:11,789 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. after waiting 0 ms 2024-12-03T16:10:11,789 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:11,789 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:11,789 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for cce9b46f903d84794859c85b38505545: Waiting for close lock at 1733242211789Disabling compacts and flushes for region at 1733242211789Disabling writes for close at 1733242211789Writing region close event to WAL at 1733242211789Closed at 1733242211789 2024-12-03T16:10:11,791 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:10:11,792 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733242211791"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242211791"}]},"ts":"1733242211791"} 2024-12-03T16:10:11,795 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:10:11,797 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:10:11,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242211797"}]},"ts":"1733242211797"} 2024-12-03T16:10:11,800 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T16:10:11,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cce9b46f903d84794859c85b38505545, ASSIGN}] 2024-12-03T16:10:11,802 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cce9b46f903d84794859c85b38505545, ASSIGN 2024-12-03T16:10:11,803 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cce9b46f903d84794859c85b38505545, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:10:11,891 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-03T16:10:11,954 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=cce9b46f903d84794859c85b38505545, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:10:11,959 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cce9b46f903d84794859c85b38505545, ASSIGN because future has completed 2024-12-03T16:10:11,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure cce9b46f903d84794859c85b38505545, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:10:11,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T16:10:12,077 INFO [master/713339a81dd9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T16:10:12,077 INFO [master/713339a81dd9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T16:10:12,119 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:12,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => cce9b46f903d84794859c85b38505545, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:10:12,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:10:12,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,122 INFO [StoreOpener-cce9b46f903d84794859c85b38505545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,125 INFO [StoreOpener-cce9b46f903d84794859c85b38505545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cce9b46f903d84794859c85b38505545 columnFamilyName testFamily 2024-12-03T16:10:12,125 DEBUG [StoreOpener-cce9b46f903d84794859c85b38505545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:10:12,126 INFO [StoreOpener-cce9b46f903d84794859c85b38505545-1 {}] regionserver.HStore(327): Store=cce9b46f903d84794859c85b38505545/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:10:12,126 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,130 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,133 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:10:12,134 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened cce9b46f903d84794859c85b38505545; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66559785, jitterRate=-0.008181914687156677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:10:12,134 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cce9b46f903d84794859c85b38505545 2024-12-03T16:10:12,135 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for cce9b46f903d84794859c85b38505545: Running coprocessor pre-open hook at 1733242212120Writing region info on filesystem at 1733242212120Initializing all the Stores at 1733242212122 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242212122Cleaning up temporary data from old regions at 1733242212128 (+6 ms)Running coprocessor post-open hooks at 1733242212134 (+6 ms)Region opened successfully at 1733242212135 (+1 ms) 2024-12-03T16:10:12,137 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545., pid=27, masterSystemTime=1733242212114 2024-12-03T16:10:12,139 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:12,139 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:12,140 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=cce9b46f903d84794859c85b38505545, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:10:12,143 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure cce9b46f903d84794859c85b38505545, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:10:12,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T16:10:12,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure cce9b46f903d84794859c85b38505545, server=713339a81dd9,44209,1733242107535 in 184 msec 2024-12-03T16:10:12,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T16:10:12,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=cce9b46f903d84794859c85b38505545, ASSIGN in 347 msec 2024-12-03T16:10:12,150 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:10:12,151 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242212150"}]},"ts":"1733242212150"} 2024-12-03T16:10:12,153 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T16:10:12,155 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:10:12,157 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.1930 sec 2024-12-03T16:10:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T16:10:14,005 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-03T16:10:14,006 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T16:10:14,006 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:14,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T16:10:14,019 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:14,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T16:10:14,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:10:14,045 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:10:14,045 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T16:10:14,050 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T16:10:14,052 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T16:10:17,622 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:10:17,655 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T16:10:17,655 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T16:10:17,892 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-03T16:10:20,051 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545., hostname=713339a81dd9,44209,1733242107535, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545., hostname=713339a81dd9,44209,1733242107535, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=713339a81dd9:44209 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-12-03T16:10:20,052 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545., hostname=713339a81dd9,44209,1733242107535, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-12-03T16:10:20,052 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545., hostname=713339a81dd9,44209,1733242107535, seqNum=2 from cache 2024-12-03T16:10:20,068 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=772 (was 848), ProcessCount=11 (was 11), AvailableMemoryMB=2018 (was 1359) - AvailableMemoryMB LEAK? - 2024-12-03T16:10:20,082 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=772, ProcessCount=11, AvailableMemoryMB=2016 2024-12-03T16:10:20,087 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:10:20,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-03T16:10:20,093 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:10:20,093 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:10:20,093 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-03T16:10:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T16:10:20,101 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:10:20,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741851_1027 (size=407) 2024-12-03T16:10:20,533 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1d79ed335e43bb52cb1091bfc578c25b, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:10:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-12-03T16:10:20,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741852_1028 (size=58) 2024-12-03T16:10:20,582 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:10:20,582 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 1d79ed335e43bb52cb1091bfc578c25b, disabling compactions & flushes 2024-12-03T16:10:20,582 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:20,582 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:20,582 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. after waiting 0 ms 2024-12-03T16:10:20,582 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:20,582 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:20,582 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1d79ed335e43bb52cb1091bfc578c25b: Waiting for close lock at 1733242220582Disabling compacts and flushes for region at 1733242220582Disabling writes for close at 1733242220582Writing region close event to WAL at 1733242220582Closed at 1733242220582 2024-12-03T16:10:20,585 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:10:20,585 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733242220585"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242220585"}]},"ts":"1733242220585"} 2024-12-03T16:10:20,589 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:10:20,591 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:10:20,591 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242220591"}]},"ts":"1733242220591"} 2024-12-03T16:10:20,595 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T16:10:20,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1d79ed335e43bb52cb1091bfc578c25b, ASSIGN}] 2024-12-03T16:10:20,597 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1d79ed335e43bb52cb1091bfc578c25b, ASSIGN 2024-12-03T16:10:20,599 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1d79ed335e43bb52cb1091bfc578c25b, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:10:20,750 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1d79ed335e43bb52cb1091bfc578c25b, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:10:20,754 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1d79ed335e43bb52cb1091bfc578c25b, ASSIGN because future has completed 2024-12-03T16:10:20,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d79ed335e43bb52cb1091bfc578c25b, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:10:20,917 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:20,917 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d79ed335e43bb52cb1091bfc578c25b, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:10:20,917 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,918 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:10:20,918 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,918 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,920 INFO [StoreOpener-1d79ed335e43bb52cb1091bfc578c25b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,923 INFO [StoreOpener-1d79ed335e43bb52cb1091bfc578c25b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d79ed335e43bb52cb1091bfc578c25b columnFamilyName testFamily 2024-12-03T16:10:20,923 DEBUG [StoreOpener-1d79ed335e43bb52cb1091bfc578c25b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:10:20,923 INFO [StoreOpener-1d79ed335e43bb52cb1091bfc578c25b-1 {}] regionserver.HStore(327): Store=1d79ed335e43bb52cb1091bfc578c25b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:10:20,924 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,925 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,926 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,926 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,928 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,932 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:10:20,934 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 1d79ed335e43bb52cb1091bfc578c25b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72699015, jitterRate=0.08329974114894867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:10:20,934 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:20,935 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 1d79ed335e43bb52cb1091bfc578c25b: Running coprocessor pre-open hook at 1733242220918Writing region info on filesystem at 1733242220918Initializing all the Stores at 1733242220920 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242220920Cleaning up temporary data from old regions at 1733242220926 (+6 ms)Running coprocessor post-open hooks at 1733242220934 (+8 ms)Region opened successfully at 1733242220935 (+1 ms) 2024-12-03T16:10:20,936 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b., pid=30, masterSystemTime=1733242220908 2024-12-03T16:10:20,941 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:20,941 INFO [RS_OPEN_PRIORITY_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:20,941 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=1d79ed335e43bb52cb1091bfc578c25b, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:10:20,945 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d79ed335e43bb52cb1091bfc578c25b, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:10:20,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T16:10:20,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d79ed335e43bb52cb1091bfc578c25b, server=713339a81dd9,44209,1733242107535 in 192 msec 2024-12-03T16:10:20,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-03T16:10:20,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=1d79ed335e43bb52cb1091bfc578c25b, ASSIGN in 356 msec 2024-12-03T16:10:20,956 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:10:20,956 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242220956"}]},"ts":"1733242220956"} 2024-12-03T16:10:20,960 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T16:10:20,962 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:10:20,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 875 msec 2024-12-03T16:10:21,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T16:10:21,115 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-03T16:10:21,115 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T16:10:21,115 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:21,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T16:10:21,121 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:21,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-03T16:10:21,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:10:21,136 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:10:21,136 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T16:10:21,141 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T16:10:21,144 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T16:10:21,146 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T16:10:21,147 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T16:10:23,894 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-03T16:10:25,911 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T16:10:26,419 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:10:27,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44209 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682828' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T16:10:27,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44209 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:54124 deadline: 1733242233152, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682828' does not exist 2024-12-03T16:10:27,155 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 713339a81dd9,44209,1733242107535 for closing scanner id = -8827874298434682828 for 1d79ed335e43bb52cb1091bfc578c25b of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8827874298434682828' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T16:10:27,161 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=221 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=734 (was 772), ProcessCount=11 (was 11), AvailableMemoryMB=1801 (was 2016) 2024-12-03T16:10:27,169 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=221, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=734, ProcessCount=11, AvailableMemoryMB=1801 2024-12-03T16:10:27,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T16:10:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T16:10:27,175 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T16:10:27,175 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:10:27,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-03T16:10:27,176 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T16:10:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T16:10:27,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741853_1029 (size=423) 2024-12-03T16:10:27,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T16:10:27,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T16:10:27,587 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 52063e1403a0a424d74990b286988d2b, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c 2024-12-03T16:10:27,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741854_1030 (size=74) 2024-12-03T16:10:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-12-03T16:10:27,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:10:27,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 52063e1403a0a424d74990b286988d2b, disabling compactions & flushes 2024-12-03T16:10:27,996 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:27,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:27,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. after waiting 0 ms 2024-12-03T16:10:27,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:27,996 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:27,996 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 52063e1403a0a424d74990b286988d2b: Waiting for close lock at 1733242227996Disabling compacts and flushes for region at 1733242227996Disabling writes for close at 1733242227996Writing region close event to WAL at 1733242227996Closed at 1733242227996 2024-12-03T16:10:27,998 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T16:10:27,998 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733242227998"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733242227998"}]},"ts":"1733242227998"} 2024-12-03T16:10:28,001 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T16:10:28,002 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T16:10:28,002 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242228002"}]},"ts":"1733242228002"} 2024-12-03T16:10:28,004 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T16:10:28,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=52063e1403a0a424d74990b286988d2b, ASSIGN}] 2024-12-03T16:10:28,006 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=52063e1403a0a424d74990b286988d2b, ASSIGN 2024-12-03T16:10:28,007 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=52063e1403a0a424d74990b286988d2b, ASSIGN; state=OFFLINE, location=713339a81dd9,44209,1733242107535; forceNewPlan=false, retain=false 2024-12-03T16:10:28,158 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=52063e1403a0a424d74990b286988d2b, regionState=OPENING, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:10:28,168 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=52063e1403a0a424d74990b286988d2b, ASSIGN because future has completed 2024-12-03T16:10:28,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52063e1403a0a424d74990b286988d2b, server=713339a81dd9,44209,1733242107535}] 2024-12-03T16:10:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T16:10:28,328 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:28,329 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 52063e1403a0a424d74990b286988d2b, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T16:10:28,329 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,329 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T16:10:28,329 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,329 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,331 INFO [StoreOpener-52063e1403a0a424d74990b286988d2b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,333 INFO [StoreOpener-52063e1403a0a424d74990b286988d2b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52063e1403a0a424d74990b286988d2b columnFamilyName testFamily 2024-12-03T16:10:28,333 DEBUG [StoreOpener-52063e1403a0a424d74990b286988d2b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T16:10:28,334 INFO [StoreOpener-52063e1403a0a424d74990b286988d2b-1 {}] regionserver.HStore(327): Store=52063e1403a0a424d74990b286988d2b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T16:10:28,334 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,335 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,335 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,336 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,336 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,338 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,341 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T16:10:28,341 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 52063e1403a0a424d74990b286988d2b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74399928, jitterRate=0.10864531993865967}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T16:10:28,341 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:28,342 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 52063e1403a0a424d74990b286988d2b: Running coprocessor pre-open hook at 1733242228329Writing region info on filesystem at 1733242228329Initializing all the Stores at 1733242228330 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733242228331 (+1 ms)Cleaning up temporary data from old regions at 1733242228336 (+5 ms)Running coprocessor post-open hooks at 1733242228341 (+5 ms)Region opened successfully at 1733242228342 (+1 ms) 2024-12-03T16:10:28,343 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., pid=33, masterSystemTime=1733242228323 2024-12-03T16:10:28,346 DEBUG [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:28,346 INFO [RS_OPEN_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:28,347 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=52063e1403a0a424d74990b286988d2b, regionState=OPEN, openSeqNum=2, regionLocation=713339a81dd9,44209,1733242107535 2024-12-03T16:10:28,349 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52063e1403a0a424d74990b286988d2b, server=713339a81dd9,44209,1733242107535 because future has completed 2024-12-03T16:10:28,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T16:10:28,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 52063e1403a0a424d74990b286988d2b, server=713339a81dd9,44209,1733242107535 in 182 msec 2024-12-03T16:10:28,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T16:10:28,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=52063e1403a0a424d74990b286988d2b, ASSIGN in 348 msec 2024-12-03T16:10:28,357 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T16:10:28,357 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733242228357"}]},"ts":"1733242228357"} 2024-12-03T16:10:28,359 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T16:10:28,360 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T16:10:28,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1890 sec 2024-12-03T16:10:29,895 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-03T16:10:30,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T16:10:30,204 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-03T16:10:30,204 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T16:10:30,205 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:30,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T16:10:30,210 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T16:10:30,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T16:10:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:10:30,224 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T16:10:30,225 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T16:10:30,228 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T16:10:30,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T16:10:31,232 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=713339a81dd9:44209 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-12-03T16:10:31,232 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-12-03T16:10:31,232 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2 from cache 2024-12-03T16:10:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-12-03T16:10:32,102 DEBUG [master/713339a81dd9:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T16:10:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2] 2024-12-03T16:10:32,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44209 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T16:10:33,231 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=713339a81dd9:44209 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=985ms, rpcTimeout=982ms 2024-12-03T16:10:33,231 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=985ms, rpcTimeout=982ms 2024-12-03T16:10:33,232 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., hostname=713339a81dd9,44209,1733242107535, seqNum=2 from cache 2024-12-03T16:10:33,241 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 221) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=446 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=652 (was 734), ProcessCount=11 (was 11), AvailableMemoryMB=1740 (was 1801) 2024-12-03T16:10:33,241 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T16:10:33,242 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T16:10:33,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:10:33,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:10:33,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T16:10:33,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T16:10:33,247 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T16:10:33,247 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T16:10:33,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:10:33,247 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:10:33,247 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T16:10:33,248 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T16:10:33,248 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1651834634, stopped=false 2024-12-03T16:10:33,248 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=713339a81dd9,37115,1733242106809 2024-12-03T16:10:33,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T16:10:33,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T16:10:33,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:10:33,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T16:10:33,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:10:33,250 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T16:10:33,251 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T16:10:33,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:10:33,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T16:10:33,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T16:10:33,251 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '713339a81dd9,44209,1733242107535' ***** 2024-12-03T16:10:33,251 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T16:10:33,251 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T16:10:33,252 INFO [RS:0;713339a81dd9:44209 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T16:10:33,252 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T16:10:33,252 INFO [RS:0;713339a81dd9:44209 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T16:10:33,252 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for 92c6012d33edc09803453b857b83ee01 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for 1d79ed335e43bb52cb1091bfc578c25b 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for 52063e1403a0a424d74990b286988d2b 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for 847db7889994e22b30682d85199e889c 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for cce9b46f903d84794859c85b38505545 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for 475f427f43d48197a370cc1148063e41 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for 482f4868465500eb74348718fb936e79 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(959): stopping server 713339a81dd9,44209,1733242107535 2024-12-03T16:10:33,253 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e5df28ab31f4c59b2cf6c16f22d2fb13, disabling compactions & flushes 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T16:10:33,253 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:10:33,253 INFO [RS:0;713339a81dd9:44209 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;713339a81dd9:44209. 2024-12-03T16:10:33,253 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:10:33,253 DEBUG [RS:0;713339a81dd9:44209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T16:10:33,253 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. after waiting 0 ms 2024-12-03T16:10:33,253 DEBUG [RS:0;713339a81dd9:44209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:10:33,253 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:10:33,254 INFO [RS:0;713339a81dd9:44209 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T16:10:33,254 INFO [RS:0;713339a81dd9:44209 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T16:10:33,254 INFO [RS:0;713339a81dd9:44209 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T16:10:33,254 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T16:10:33,254 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-03T16:10:33,254 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T16:10:33,254 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T16:10:33,254 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T16:10:33,254 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T16:10:33,254 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T16:10:33,254 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d79ed335e43bb52cb1091bfc578c25b, 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, 92c6012d33edc09803453b857b83ee01, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289, e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:10:33,257 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e5df28ab31f4c59b2cf6c16f22d2fb13 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:33,257 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-03T16:10:33,319 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13/.tmp/testFamily/d823f619e70941399e11c6146200085f is 61, key is row-0/testFamily:testQualifier/1733242173329/Put/seqid=0 2024-12-03T16:10:33,321 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/info/989c2ebbe00948549b3b4cb6f8aa6720 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13./info:regioninfo/1733242173101/Put/seqid=0 2024-12-03T16:10:33,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741856_1032 (size=5049) 2024-12-03T16:10:33,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741855_1031 (size=20564) 2024-12-03T16:10:33,455 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d79ed335e43bb52cb1091bfc578c25b, 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, 92c6012d33edc09803453b857b83ee01, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289, e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:10:33,469 INFO [regionserver/713339a81dd9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T16:10:33,470 INFO [regionserver/713339a81dd9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T16:10:33,655 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d79ed335e43bb52cb1091bfc578c25b, 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, 92c6012d33edc09803453b857b83ee01, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289, e5df28ab31f4c59b2cf6c16f22d2fb13 2024-12-03T16:10:33,733 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13/.tmp/testFamily/d823f619e70941399e11c6146200085f 2024-12-03T16:10:33,735 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/info/989c2ebbe00948549b3b4cb6f8aa6720 2024-12-03T16:10:33,748 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-03T16:10:33,782 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13/.tmp/testFamily/d823f619e70941399e11c6146200085f as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13/testFamily/d823f619e70941399e11c6146200085f 2024-12-03T16:10:33,793 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13/testFamily/d823f619e70941399e11c6146200085f, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:33,799 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e5df28ab31f4c59b2cf6c16f22d2fb13 in 541ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:33,799 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/ns/7f6323f4306049c39d48341c8a7c4149 is 43, key is default/ns:d/1733242171970/Put/seqid=0 2024-12-03T16:10:33,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741857_1033 (size=5153) 2024-12-03T16:10:33,807 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextExceptionAsync/e5df28ab31f4c59b2cf6c16f22d2fb13/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:33,809 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:10:33,809 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e5df28ab31f4c59b2cf6c16f22d2fb13: Waiting for close lock at 1733242233253Running coprocessor pre-close hooks at 1733242233253Disabling compacts and flushes for region at 1733242233253Disabling writes for close at 1733242233253Obtaining lock to block concurrent updates at 1733242233257 (+4 ms)Preparing flush snapshotting stores in e5df28ab31f4c59b2cf6c16f22d2fb13 at 1733242233257Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242233265 (+8 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. at 1733242233266 (+1 ms)Flushing e5df28ab31f4c59b2cf6c16f22d2fb13/testFamily: creating writer at 1733242233267 (+1 ms)Flushing e5df28ab31f4c59b2cf6c16f22d2fb13/testFamily: appending metadata at 1733242233312 (+45 ms)Flushing e5df28ab31f4c59b2cf6c16f22d2fb13/testFamily: closing flushed file at 1733242233315 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65508fa5: reopening flushed file at 1733242233780 (+465 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e5df28ab31f4c59b2cf6c16f22d2fb13 in 541ms, sequenceid=8, compaction requested=false at 1733242233799 (+19 ms)Writing region close event to WAL at 1733242233801 (+2 ms)Running coprocessor post-close hooks at 1733242233807 (+6 ms)Closed at 1733242233809 (+2 ms) 2024-12-03T16:10:33,810 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733242172232.e5df28ab31f4c59b2cf6c16f22d2fb13. 2024-12-03T16:10:33,810 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 92c6012d33edc09803453b857b83ee01, disabling compactions & flushes 2024-12-03T16:10:33,810 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:33,810 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:33,810 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. after waiting 0 ms 2024-12-03T16:10:33,810 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:33,810 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 92c6012d33edc09803453b857b83ee01 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:33,828 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01/.tmp/testFamily/98c300da55e34ed6a0e3138a7e9c4079 is 61, key is row-0/testFamily:testQualifier/1733242207925/Put/seqid=0 2024-12-03T16:10:33,831 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:10:33,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741858_1034 (size=5049) 2024-12-03T16:10:33,855 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d79ed335e43bb52cb1091bfc578c25b, 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, 92c6012d33edc09803453b857b83ee01, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:34,056 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d79ed335e43bb52cb1091bfc578c25b, 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, 92c6012d33edc09803453b857b83ee01, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:34,206 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/ns/7f6323f4306049c39d48341c8a7c4149 2024-12-03T16:10:34,232 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/table/7105f9d7e3604218a6922f48bca3fd64 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733242173116/Put/seqid=0 2024-12-03T16:10:34,234 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01/.tmp/testFamily/98c300da55e34ed6a0e3138a7e9c4079 2024-12-03T16:10:34,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741859_1035 (size=6699) 2024-12-03T16:10:34,241 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/table/7105f9d7e3604218a6922f48bca3fd64 2024-12-03T16:10:34,245 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01/.tmp/testFamily/98c300da55e34ed6a0e3138a7e9c4079 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01/testFamily/98c300da55e34ed6a0e3138a7e9c4079 2024-12-03T16:10:34,250 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/info/989c2ebbe00948549b3b4cb6f8aa6720 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/info/989c2ebbe00948549b3b4cb6f8aa6720 2024-12-03T16:10:34,254 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01/testFamily/98c300da55e34ed6a0e3138a7e9c4079, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:34,256 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-12-03T16:10:34,256 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1d79ed335e43bb52cb1091bfc578c25b, 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, 92c6012d33edc09803453b857b83ee01, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:34,257 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 92c6012d33edc09803453b857b83ee01 in 447ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:34,258 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/info/989c2ebbe00948549b3b4cb6f8aa6720, entries=100, sequenceid=56, filesize=20.1 K 2024-12-03T16:10:34,260 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/ns/7f6323f4306049c39d48341c8a7c4149 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/ns/7f6323f4306049c39d48341c8a7c4149 2024-12-03T16:10:34,263 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNext/92c6012d33edc09803453b857b83ee01/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:34,264 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:34,264 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 92c6012d33edc09803453b857b83ee01: Waiting for close lock at 1733242233810Running coprocessor pre-close hooks at 1733242233810Disabling compacts and flushes for region at 1733242233810Disabling writes for close at 1733242233810Obtaining lock to block concurrent updates at 1733242233810Preparing flush snapshotting stores in 92c6012d33edc09803453b857b83ee01 at 1733242233810Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242233811 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. at 1733242233811Flushing 92c6012d33edc09803453b857b83ee01/testFamily: creating writer at 1733242233811Flushing 92c6012d33edc09803453b857b83ee01/testFamily: appending metadata at 1733242233828 (+17 ms)Flushing 92c6012d33edc09803453b857b83ee01/testFamily: closing flushed file at 1733242233828Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@754c6570: reopening flushed file at 1733242234244 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 92c6012d33edc09803453b857b83ee01 in 447ms, sequenceid=8, compaction requested=false at 1733242234257 (+13 ms)Writing region close event to WAL at 1733242234258 (+1 ms)Running coprocessor post-close hooks at 1733242234264 (+6 ms)Closed at 1733242234264 2024-12-03T16:10:34,265 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733242204872.92c6012d33edc09803453b857b83ee01. 2024-12-03T16:10:34,265 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1d79ed335e43bb52cb1091bfc578c25b, disabling compactions & flushes 2024-12-03T16:10:34,265 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:34,265 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:34,265 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. after waiting 0 ms 2024-12-03T16:10:34,265 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:34,265 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1d79ed335e43bb52cb1091bfc578c25b 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:34,269 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/ns/7f6323f4306049c39d48341c8a7c4149, entries=2, sequenceid=56, filesize=5.0 K 2024-12-03T16:10:34,270 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/.tmp/table/7105f9d7e3604218a6922f48bca3fd64 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/table/7105f9d7e3604218a6922f48bca3fd64 2024-12-03T16:10:34,279 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/table/7105f9d7e3604218a6922f48bca3fd64, entries=20, sequenceid=56, filesize=6.5 K 2024-12-03T16:10:34,281 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1027ms, sequenceid=56, compaction requested=false 2024-12-03T16:10:34,284 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b/.tmp/testFamily/1c32e43ebb4a47b191e5cf4f6d609521 is 61, key is row-0/testFamily:testQualifier/1733242221126/Put/seqid=0 2024-12-03T16:10:34,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741860_1036 (size=5049) 2024-12-03T16:10:34,290 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-03T16:10:34,291 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T16:10:34,291 INFO [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T16:10:34,292 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733242233254Running coprocessor pre-close hooks at 1733242233254Disabling compacts and flushes for region at 1733242233254Disabling writes for close at 1733242233254Obtaining lock to block concurrent updates at 1733242233257 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733242233257Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733242233265 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733242233266 (+1 ms)Flushing 1588230740/info: creating writer at 1733242233267 (+1 ms)Flushing 1588230740/info: appending metadata at 1733242233318 (+51 ms)Flushing 1588230740/info: closing flushed file at 1733242233318Flushing 1588230740/ns: creating writer at 1733242233780 (+462 ms)Flushing 1588230740/ns: appending metadata at 1733242233798 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733242233798Flushing 1588230740/table: creating writer at 1733242234215 (+417 ms)Flushing 1588230740/table: appending metadata at 1733242234231 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733242234231Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d07957b: reopening flushed file at 1733242234249 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e3d9b1: reopening flushed file at 1733242234258 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b5a892d: reopening flushed file at 1733242234269 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1027ms, sequenceid=56, compaction requested=false at 1733242234281 (+12 ms)Writing region close event to WAL at 1733242234285 (+4 ms)Running coprocessor post-close hooks at 1733242234291 (+6 ms)Closed at 1733242234291 2024-12-03T16:10:34,292 DEBUG [RS_CLOSE_META-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T16:10:34,456 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 1d79ed335e43bb52cb1091bfc578c25b, 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:34,657 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 1d79ed335e43bb52cb1091bfc578c25b, 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:34,691 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b/.tmp/testFamily/1c32e43ebb4a47b191e5cf4f6d609521 2024-12-03T16:10:34,701 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b/.tmp/testFamily/1c32e43ebb4a47b191e5cf4f6d609521 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b/testFamily/1c32e43ebb4a47b191e5cf4f6d609521 2024-12-03T16:10:34,708 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b/testFamily/1c32e43ebb4a47b191e5cf4f6d609521, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:34,709 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1d79ed335e43bb52cb1091bfc578c25b in 444ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:34,714 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNext/1d79ed335e43bb52cb1091bfc578c25b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:34,715 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:34,715 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1d79ed335e43bb52cb1091bfc578c25b: Waiting for close lock at 1733242234265Running coprocessor pre-close hooks at 1733242234265Disabling compacts and flushes for region at 1733242234265Disabling writes for close at 1733242234265Obtaining lock to block concurrent updates at 1733242234265Preparing flush snapshotting stores in 1d79ed335e43bb52cb1091bfc578c25b at 1733242234265Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242234266 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. at 1733242234266Flushing 1d79ed335e43bb52cb1091bfc578c25b/testFamily: creating writer at 1733242234266Flushing 1d79ed335e43bb52cb1091bfc578c25b/testFamily: appending metadata at 1733242234284 (+18 ms)Flushing 1d79ed335e43bb52cb1091bfc578c25b/testFamily: closing flushed file at 1733242234284Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26998fa7: reopening flushed file at 1733242234699 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1d79ed335e43bb52cb1091bfc578c25b in 444ms, sequenceid=8, compaction requested=false at 1733242234709 (+10 ms)Writing region close event to WAL at 1733242234710 (+1 ms)Running coprocessor post-close hooks at 1733242234715 (+5 ms)Closed at 1733242234715 2024-12-03T16:10:34,715 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733242220085.1d79ed335e43bb52cb1091bfc578c25b. 2024-12-03T16:10:34,715 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 52063e1403a0a424d74990b286988d2b, disabling compactions & flushes 2024-12-03T16:10:34,715 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:34,716 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:34,716 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. after waiting 0 ms 2024-12-03T16:10:34,716 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:34,716 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 52063e1403a0a424d74990b286988d2b 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:34,733 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b/.tmp/testFamily/b3e9eaa1dd0d4c32a109d8c6501af7c5 is 61, key is row-0/testFamily:testQualifier/1733242230215/Put/seqid=0 2024-12-03T16:10:34,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741861_1037 (size=5049) 2024-12-03T16:10:34,857 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:35,057 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 52063e1403a0a424d74990b286988d2b, 847db7889994e22b30682d85199e889c, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:35,139 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b/.tmp/testFamily/b3e9eaa1dd0d4c32a109d8c6501af7c5 2024-12-03T16:10:35,149 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b/.tmp/testFamily/b3e9eaa1dd0d4c32a109d8c6501af7c5 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b/testFamily/b3e9eaa1dd0d4c32a109d8c6501af7c5 2024-12-03T16:10:35,157 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b/testFamily/b3e9eaa1dd0d4c32a109d8c6501af7c5, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:35,158 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 52063e1403a0a424d74990b286988d2b in 442ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:35,163 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScannerAsync/52063e1403a0a424d74990b286988d2b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:35,164 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:35,164 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 52063e1403a0a424d74990b286988d2b: Waiting for close lock at 1733242234715Running coprocessor pre-close hooks at 1733242234715Disabling compacts and flushes for region at 1733242234715Disabling writes for close at 1733242234716 (+1 ms)Obtaining lock to block concurrent updates at 1733242234716Preparing flush snapshotting stores in 52063e1403a0a424d74990b286988d2b at 1733242234716Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242234716Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. at 1733242234717 (+1 ms)Flushing 52063e1403a0a424d74990b286988d2b/testFamily: creating writer at 1733242234717Flushing 52063e1403a0a424d74990b286988d2b/testFamily: appending metadata at 1733242234733 (+16 ms)Flushing 52063e1403a0a424d74990b286988d2b/testFamily: closing flushed file at 1733242234733Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62d0ad9a: reopening flushed file at 1733242235148 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 52063e1403a0a424d74990b286988d2b in 442ms, sequenceid=8, compaction requested=false at 1733242235158 (+10 ms)Writing region close event to WAL at 1733242235159 (+1 ms)Running coprocessor post-close hooks at 1733242235164 (+5 ms)Closed at 1733242235164 2024-12-03T16:10:35,164 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733242227171.52063e1403a0a424d74990b286988d2b. 2024-12-03T16:10:35,164 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 847db7889994e22b30682d85199e889c, disabling compactions & flushes 2024-12-03T16:10:35,164 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:10:35,164 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:10:35,164 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. after waiting 0 ms 2024-12-03T16:10:35,164 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:10:35,164 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 847db7889994e22b30682d85199e889c 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:35,182 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c/.tmp/testFamily/b8bd364d127e4294a5809e4bcd174326 is 61, key is row-0/testFamily:testQualifier/1733242192646/Put/seqid=0 2024-12-03T16:10:35,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741862_1038 (size=5049) 2024-12-03T16:10:35,257 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-03T16:10:35,258 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1325): Online Regions={847db7889994e22b30682d85199e889c=testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c., cce9b46f903d84794859c85b38505545=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545., 475f427f43d48197a370cc1148063e41=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., dc0fbe6bdf3499b5dceba341c33d4289=testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289., 482f4868465500eb74348718fb936e79=hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79., 3db5b4cbdc1bdb1a2d5113b613ab8786=hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786.} 2024-12-03T16:10:35,258 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 847db7889994e22b30682d85199e889c, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:35,458 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, 847db7889994e22b30682d85199e889c, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:35,588 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c/.tmp/testFamily/b8bd364d127e4294a5809e4bcd174326 2024-12-03T16:10:35,597 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c/.tmp/testFamily/b8bd364d127e4294a5809e4bcd174326 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c/testFamily/b8bd364d127e4294a5809e4bcd174326 2024-12-03T16:10:35,606 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c/testFamily/b8bd364d127e4294a5809e4bcd174326, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:35,608 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 847db7889994e22b30682d85199e889c in 443ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:35,612 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testRetryOutOfOrderScannerNextException/847db7889994e22b30682d85199e889c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:35,613 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:10:35,613 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 847db7889994e22b30682d85199e889c: Waiting for close lock at 1733242235164Running coprocessor pre-close hooks at 1733242235164Disabling compacts and flushes for region at 1733242235164Disabling writes for close at 1733242235164Obtaining lock to block concurrent updates at 1733242235164Preparing flush snapshotting stores in 847db7889994e22b30682d85199e889c at 1733242235164Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242235165 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. at 1733242235165Flushing 847db7889994e22b30682d85199e889c/testFamily: creating writer at 1733242235165Flushing 847db7889994e22b30682d85199e889c/testFamily: appending metadata at 1733242235182 (+17 ms)Flushing 847db7889994e22b30682d85199e889c/testFamily: closing flushed file at 1733242235182Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fe000bc: reopening flushed file at 1733242235596 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 847db7889994e22b30682d85199e889c in 443ms, sequenceid=8, compaction requested=false at 1733242235608 (+12 ms)Writing region close event to WAL at 1733242235608Running coprocessor post-close hooks at 1733242235613 (+5 ms)Closed at 1733242235613 2024-12-03T16:10:35,613 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733242191617.847db7889994e22b30682d85199e889c. 2024-12-03T16:10:35,614 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cce9b46f903d84794859c85b38505545, disabling compactions & flushes 2024-12-03T16:10:35,614 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:35,614 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:35,614 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. after waiting 0 ms 2024-12-03T16:10:35,614 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:35,614 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cce9b46f903d84794859c85b38505545 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:35,632 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545/.tmp/testFamily/0c47b198298d4f0d80f0c067e8102a2d is 61, key is row-0/testFamily:testQualifier/1733242214029/Put/seqid=0 2024-12-03T16:10:35,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741863_1039 (size=5049) 2024-12-03T16:10:35,658 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:35,859 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:36,038 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545/.tmp/testFamily/0c47b198298d4f0d80f0c067e8102a2d 2024-12-03T16:10:36,049 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545/.tmp/testFamily/0c47b198298d4f0d80f0c067e8102a2d as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545/testFamily/0c47b198298d4f0d80f0c067e8102a2d 2024-12-03T16:10:36,059 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, cce9b46f903d84794859c85b38505545, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:36,059 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545/testFamily/0c47b198298d4f0d80f0c067e8102a2d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:36,060 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cce9b46f903d84794859c85b38505545 in 446ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:36,066 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/cce9b46f903d84794859c85b38505545/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:36,066 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:36,067 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cce9b46f903d84794859c85b38505545: Waiting for close lock at 1733242235614Running coprocessor pre-close hooks at 1733242235614Disabling compacts and flushes for region at 1733242235614Disabling writes for close at 1733242235614Obtaining lock to block concurrent updates at 1733242235614Preparing flush snapshotting stores in cce9b46f903d84794859c85b38505545 at 1733242235614Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242235614Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. at 1733242235615 (+1 ms)Flushing cce9b46f903d84794859c85b38505545/testFamily: creating writer at 1733242235615Flushing cce9b46f903d84794859c85b38505545/testFamily: appending metadata at 1733242235632 (+17 ms)Flushing cce9b46f903d84794859c85b38505545/testFamily: closing flushed file at 1733242235632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52aec999: reopening flushed file at 1733242236047 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cce9b46f903d84794859c85b38505545 in 446ms, sequenceid=8, compaction requested=false at 1733242236060 (+13 ms)Writing region close event to WAL at 1733242236061 (+1 ms)Running coprocessor post-close hooks at 1733242236066 (+5 ms)Closed at 1733242236066 2024-12-03T16:10:36,067 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733242210961.cce9b46f903d84794859c85b38505545. 2024-12-03T16:10:36,067 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 475f427f43d48197a370cc1148063e41, disabling compactions & flushes 2024-12-03T16:10:36,067 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:10:36,067 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:10:36,067 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. after waiting 0 ms 2024-12-03T16:10:36,067 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:10:36,067 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 475f427f43d48197a370cc1148063e41 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:36,085 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41/.tmp/testFamily/88156005abb14ec6b2aec25d922ebf16 is 61, key is row-0/testFamily:testQualifier/1733242195745/Put/seqid=0 2024-12-03T16:10:36,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741864_1040 (size=5049) 2024-12-03T16:10:36,259 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-12-03T16:10:36,259 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1325): Online Regions={475f427f43d48197a370cc1148063e41=testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., dc0fbe6bdf3499b5dceba341c33d4289=testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289., 482f4868465500eb74348718fb936e79=hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79., 3db5b4cbdc1bdb1a2d5113b613ab8786=hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786.} 2024-12-03T16:10:36,260 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:36,379 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=713339a81dd9:37115 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-12-03T16:10:36,460 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 475f427f43d48197a370cc1148063e41, 482f4868465500eb74348718fb936e79, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:36,494 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41/.tmp/testFamily/88156005abb14ec6b2aec25d922ebf16 2024-12-03T16:10:36,503 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41/.tmp/testFamily/88156005abb14ec6b2aec25d922ebf16 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41/testFamily/88156005abb14ec6b2aec25d922ebf16 2024-12-03T16:10:36,511 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41/testFamily/88156005abb14ec6b2aec25d922ebf16, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:36,512 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 475f427f43d48197a370cc1148063e41 in 445ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:36,518 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnOpenScanner/475f427f43d48197a370cc1148063e41/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:36,518 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:10:36,519 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 475f427f43d48197a370cc1148063e41: Waiting for close lock at 1733242236067Running coprocessor pre-close hooks at 1733242236067Disabling compacts and flushes for region at 1733242236067Disabling writes for close at 1733242236067Obtaining lock to block concurrent updates at 1733242236067Preparing flush snapshotting stores in 475f427f43d48197a370cc1148063e41 at 1733242236067Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242236067Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. at 1733242236068 (+1 ms)Flushing 475f427f43d48197a370cc1148063e41/testFamily: creating writer at 1733242236068Flushing 475f427f43d48197a370cc1148063e41/testFamily: appending metadata at 1733242236084 (+16 ms)Flushing 475f427f43d48197a370cc1148063e41/testFamily: closing flushed file at 1733242236084Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ce7f8e9: reopening flushed file at 1733242236502 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 475f427f43d48197a370cc1148063e41 in 445ms, sequenceid=8, compaction requested=false at 1733242236512 (+10 ms)Writing region close event to WAL at 1733242236513 (+1 ms)Running coprocessor post-close hooks at 1733242236518 (+5 ms)Closed at 1733242236518 2024-12-03T16:10:36,519 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733242192709.475f427f43d48197a370cc1148063e41. 2024-12-03T16:10:36,519 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc0fbe6bdf3499b5dceba341c33d4289, disabling compactions & flushes 2024-12-03T16:10:36,519 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:10:36,519 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:10:36,519 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. after waiting 0 ms 2024-12-03T16:10:36,519 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:10:36,519 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dc0fbe6bdf3499b5dceba341c33d4289 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:36,537 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289/.tmp/testFamily/66a37feda0f644e2863724f5e6b87c77 is 61, key is row-0/testFamily:testQualifier/1733242201826/Put/seqid=0 2024-12-03T16:10:36,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741865_1041 (size=5049) 2024-12-03T16:10:36,660 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 482f4868465500eb74348718fb936e79, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:36,861 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 482f4868465500eb74348718fb936e79, dc0fbe6bdf3499b5dceba341c33d4289 2024-12-03T16:10:36,943 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289/.tmp/testFamily/66a37feda0f644e2863724f5e6b87c77 2024-12-03T16:10:36,952 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289/.tmp/testFamily/66a37feda0f644e2863724f5e6b87c77 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289/testFamily/66a37feda0f644e2863724f5e6b87c77 2024-12-03T16:10:36,961 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289/testFamily/66a37feda0f644e2863724f5e6b87c77, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:36,963 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dc0fbe6bdf3499b5dceba341c33d4289 in 444ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:36,969 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/default/testNormalScanTimeoutOnNextAsync/dc0fbe6bdf3499b5dceba341c33d4289/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:36,970 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:10:36,970 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc0fbe6bdf3499b5dceba341c33d4289: Waiting for close lock at 1733242236519Running coprocessor pre-close hooks at 1733242236519Disabling compacts and flushes for region at 1733242236519Disabling writes for close at 1733242236519Obtaining lock to block concurrent updates at 1733242236519Preparing flush snapshotting stores in dc0fbe6bdf3499b5dceba341c33d4289 at 1733242236519Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242236519Flushing stores of testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. at 1733242236520 (+1 ms)Flushing dc0fbe6bdf3499b5dceba341c33d4289/testFamily: creating writer at 1733242236520Flushing dc0fbe6bdf3499b5dceba341c33d4289/testFamily: appending metadata at 1733242236536 (+16 ms)Flushing dc0fbe6bdf3499b5dceba341c33d4289/testFamily: closing flushed file at 1733242236536Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6334a200: reopening flushed file at 1733242236951 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for dc0fbe6bdf3499b5dceba341c33d4289 in 444ms, sequenceid=8, compaction requested=false at 1733242236963 (+12 ms)Writing region close event to WAL at 1733242236964 (+1 ms)Running coprocessor post-close hooks at 1733242236969 (+5 ms)Closed at 1733242236969 2024-12-03T16:10:36,970 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733242198792.dc0fbe6bdf3499b5dceba341c33d4289. 2024-12-03T16:10:36,970 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 482f4868465500eb74348718fb936e79, disabling compactions & flushes 2024-12-03T16:10:36,970 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:10:36,970 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:10:36,970 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. after waiting 0 ms 2024-12-03T16:10:36,971 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:10:36,971 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 482f4868465500eb74348718fb936e79 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:36,995 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79/.tmp/testFamily/e92fde6d143545bcb632c357154a7a2d is 61, key is row-0/testFamily:testQualifier/1733242185565/Put/seqid=0 2024-12-03T16:10:37,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741866_1042 (size=5049) 2024-12-03T16:10:37,061 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 482f4868465500eb74348718fb936e79 2024-12-03T16:10:37,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T16:10:37,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T16:10:37,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T16:10:37,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T16:10:37,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T16:10:37,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T16:10:37,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T16:10:37,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T16:10:37,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T16:10:37,261 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T16:10:37,261 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1325): Online Regions={482f4868465500eb74348718fb936e79=hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79., 3db5b4cbdc1bdb1a2d5113b613ab8786=hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786.} 2024-12-03T16:10:37,261 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786, 482f4868465500eb74348718fb936e79 2024-12-03T16:10:37,402 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79/.tmp/testFamily/e92fde6d143545bcb632c357154a7a2d 2024-12-03T16:10:37,412 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79/.tmp/testFamily/e92fde6d143545bcb632c357154a7a2d as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79/testFamily/e92fde6d143545bcb632c357154a7a2d 2024-12-03T16:10:37,421 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79/testFamily/e92fde6d143545bcb632c357154a7a2d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:37,422 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 482f4868465500eb74348718fb936e79 in 451ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:37,428 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnNextAsync/482f4868465500eb74348718fb936e79/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:37,429 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:10:37,429 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 482f4868465500eb74348718fb936e79: Waiting for close lock at 1733242236970Running coprocessor pre-close hooks at 1733242236970Disabling compacts and flushes for region at 1733242236970Disabling writes for close at 1733242236970Obtaining lock to block concurrent updates at 1733242236971 (+1 ms)Preparing flush snapshotting stores in 482f4868465500eb74348718fb936e79 at 1733242236971Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242236971Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. at 1733242236972 (+1 ms)Flushing 482f4868465500eb74348718fb936e79/testFamily: creating writer at 1733242236972Flushing 482f4868465500eb74348718fb936e79/testFamily: appending metadata at 1733242236995 (+23 ms)Flushing 482f4868465500eb74348718fb936e79/testFamily: closing flushed file at 1733242236995Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1abbb44a: reopening flushed file at 1733242237411 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 482f4868465500eb74348718fb936e79 in 451ms, sequenceid=8, compaction requested=false at 1733242237422 (+11 ms)Writing region close event to WAL at 1733242237423 (+1 ms)Running coprocessor post-close hooks at 1733242237429 (+6 ms)Closed at 1733242237429 2024-12-03T16:10:37,429 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733242182512.482f4868465500eb74348718fb936e79. 2024-12-03T16:10:37,430 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3db5b4cbdc1bdb1a2d5113b613ab8786, disabling compactions & flushes 2024-12-03T16:10:37,430 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:10:37,430 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:10:37,430 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. after waiting 0 ms 2024-12-03T16:10:37,430 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:10:37,430 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3db5b4cbdc1bdb1a2d5113b613ab8786 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T16:10:37,456 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786/.tmp/testFamily/e7714c00299b4ddca85306d63013e3bc is 61, key is row-0/testFamily:testQualifier/1733242176458/Put/seqid=0 2024-12-03T16:10:37,462 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:10:37,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741867_1043 (size=5049) 2024-12-03T16:10:37,662 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:10:37,862 DEBUG [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1351): Waiting on 3db5b4cbdc1bdb1a2d5113b613ab8786 2024-12-03T16:10:37,863 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786/.tmp/testFamily/e7714c00299b4ddca85306d63013e3bc 2024-12-03T16:10:37,872 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786/.tmp/testFamily/e7714c00299b4ddca85306d63013e3bc as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786/testFamily/e7714c00299b4ddca85306d63013e3bc 2024-12-03T16:10:37,879 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786/testFamily/e7714c00299b4ddca85306d63013e3bc, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T16:10:37,880 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3db5b4cbdc1bdb1a2d5113b613ab8786 in 450ms, sequenceid=8, compaction requested=false 2024-12-03T16:10:37,885 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/data/hbase/testMetaScanTimeoutOnOpenScanner/3db5b4cbdc1bdb1a2d5113b613ab8786/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T16:10:37,886 INFO [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:10:37,886 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3db5b4cbdc1bdb1a2d5113b613ab8786: Waiting for close lock at 1733242237430Running coprocessor pre-close hooks at 1733242237430Disabling compacts and flushes for region at 1733242237430Disabling writes for close at 1733242237430Obtaining lock to block concurrent updates at 1733242237430Preparing flush snapshotting stores in 3db5b4cbdc1bdb1a2d5113b613ab8786 at 1733242237430Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733242237430Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. at 1733242237431 (+1 ms)Flushing 3db5b4cbdc1bdb1a2d5113b613ab8786/testFamily: creating writer at 1733242237431Flushing 3db5b4cbdc1bdb1a2d5113b613ab8786/testFamily: appending metadata at 1733242237455 (+24 ms)Flushing 3db5b4cbdc1bdb1a2d5113b613ab8786/testFamily: closing flushed file at 1733242237455Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@433f8316: reopening flushed file at 1733242237871 (+416 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3db5b4cbdc1bdb1a2d5113b613ab8786 in 450ms, sequenceid=8, compaction requested=false at 1733242237880 (+9 ms)Writing region close event to WAL at 1733242237881 (+1 ms)Running coprocessor post-close hooks at 1733242237886 (+5 ms)Closed at 1733242237886 2024-12-03T16:10:37,886 DEBUG [RS_CLOSE_REGION-regionserver/713339a81dd9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733242173416.3db5b4cbdc1bdb1a2d5113b613ab8786. 2024-12-03T16:10:38,063 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(976): stopping server 713339a81dd9,44209,1733242107535; all regions closed. 2024-12-03T16:10:38,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741834_1010 (size=19479) 2024-12-03T16:10:38,074 DEBUG [RS:0;713339a81dd9:44209 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/oldWALs 2024-12-03T16:10:38,074 INFO [RS:0;713339a81dd9:44209 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 713339a81dd9%2C44209%2C1733242107535.meta:.meta(num 1733242171626) 2024-12-03T16:10:38,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741833_1009 (size=20031) 2024-12-03T16:10:38,079 DEBUG [RS:0;713339a81dd9:44209 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/oldWALs 2024-12-03T16:10:38,079 INFO [RS:0;713339a81dd9:44209 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 713339a81dd9%2C44209%2C1733242107535:(num 1733242171285) 2024-12-03T16:10:38,079 DEBUG [RS:0;713339a81dd9:44209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T16:10:38,079 INFO [RS:0;713339a81dd9:44209 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T16:10:38,079 INFO [RS:0;713339a81dd9:44209 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T16:10:38,079 INFO [RS:0;713339a81dd9:44209 {}] hbase.ChoreService(370): Chore service for: regionserver/713339a81dd9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T16:10:38,080 INFO [RS:0;713339a81dd9:44209 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T16:10:38,080 INFO [regionserver/713339a81dd9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T16:10:38,081 INFO [RS:0;713339a81dd9:44209 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44209 2024-12-03T16:10:38,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T16:10:38,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/713339a81dd9,44209,1733242107535 2024-12-03T16:10:38,085 INFO [RS:0;713339a81dd9:44209 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T16:10:38,088 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [713339a81dd9,44209,1733242107535] 2024-12-03T16:10:38,089 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/713339a81dd9,44209,1733242107535 already deleted, retry=false 2024-12-03T16:10:38,089 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 713339a81dd9,44209,1733242107535 expired; onlineServers=0 2024-12-03T16:10:38,089 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '713339a81dd9,37115,1733242106809' ***** 2024-12-03T16:10:38,089 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T16:10:38,090 INFO [M:0;713339a81dd9:37115 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T16:10:38,090 INFO [M:0;713339a81dd9:37115 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T16:10:38,090 DEBUG [M:0;713339a81dd9:37115 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T16:10:38,090 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T16:10:38,090 DEBUG [M:0;713339a81dd9:37115 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T16:10:38,090 DEBUG [master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.large.0-1733242109480 {}] cleaner.HFileCleaner(306): Exit Thread[master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.large.0-1733242109480,5,FailOnTimeoutGroup] 2024-12-03T16:10:38,090 DEBUG [master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.small.0-1733242109483 {}] cleaner.HFileCleaner(306): Exit Thread[master/713339a81dd9:0:becomeActiveMaster-HFileCleaner.small.0-1733242109483,5,FailOnTimeoutGroup] 2024-12-03T16:10:38,090 INFO [M:0;713339a81dd9:37115 {}] hbase.ChoreService(370): Chore service for: master/713339a81dd9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T16:10:38,090 INFO [M:0;713339a81dd9:37115 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T16:10:38,091 DEBUG [M:0;713339a81dd9:37115 {}] master.HMaster(1795): Stopping service threads 2024-12-03T16:10:38,091 INFO [M:0;713339a81dd9:37115 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T16:10:38,091 INFO [M:0;713339a81dd9:37115 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T16:10:38,091 INFO [M:0;713339a81dd9:37115 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T16:10:38,092 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T16:10:38,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T16:10:38,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T16:10:38,093 DEBUG [M:0;713339a81dd9:37115 {}] zookeeper.ZKUtil(347): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T16:10:38,093 WARN [M:0;713339a81dd9:37115 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T16:10:38,094 INFO [M:0;713339a81dd9:37115 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/.lastflushedseqids 2024-12-03T16:10:38,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741868_1044 (size=712) 2024-12-03T16:10:38,175 INFO [regionserver/713339a81dd9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T16:10:38,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T16:10:38,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44209-0x101793116f70001, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T16:10:38,188 INFO [RS:0;713339a81dd9:44209 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T16:10:38,189 INFO [RS:0;713339a81dd9:44209 {}] regionserver.HRegionServer(1031): Exiting; stopping=713339a81dd9,44209,1733242107535; zookeeper connection closed. 2024-12-03T16:10:38,189 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f4cf5d1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f4cf5d1 2024-12-03T16:10:38,190 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T16:10:38,509 INFO [M:0;713339a81dd9:37115 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T16:10:38,509 INFO [M:0;713339a81dd9:37115 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T16:10:38,509 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T16:10:38,509 INFO [M:0;713339a81dd9:37115 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T16:10:38,509 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T16:10:38,509 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T16:10:38,509 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T16:10:38,510 INFO [M:0;713339a81dd9:37115 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.05 KB heapSize=191.12 KB 2024-12-03T16:10:38,528 DEBUG [M:0;713339a81dd9:37115 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ea18192b38d8424fbdad54a887706b0a is 82, key is hbase:meta,,1/info:regioninfo/1733242171730/Put/seqid=0 2024-12-03T16:10:38,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741869_1045 (size=5672) 2024-12-03T16:10:38,933 INFO [M:0;713339a81dd9:37115 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ea18192b38d8424fbdad54a887706b0a 2024-12-03T16:10:38,959 DEBUG [M:0;713339a81dd9:37115 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d82aa8c07b3489abba0ff9447455858 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733242173123/Put/seqid=0 2024-12-03T16:10:38,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741870_1046 (size=14067) 2024-12-03T16:10:39,311 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T16:10:39,365 INFO [M:0;713339a81dd9:37115 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.50 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d82aa8c07b3489abba0ff9447455858 2024-12-03T16:10:39,372 INFO [M:0;713339a81dd9:37115 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3d82aa8c07b3489abba0ff9447455858 2024-12-03T16:10:39,390 DEBUG [M:0;713339a81dd9:37115 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38420afa4d35406bb8f4ced34677044d is 69, key is 713339a81dd9,44209,1733242107535/rs:state/1733242139615/Put/seqid=0 2024-12-03T16:10:39,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741871_1047 (size=5156) 2024-12-03T16:10:39,796 INFO [M:0;713339a81dd9:37115 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38420afa4d35406bb8f4ced34677044d 2024-12-03T16:10:39,805 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ea18192b38d8424fbdad54a887706b0a as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ea18192b38d8424fbdad54a887706b0a 2024-12-03T16:10:39,813 INFO [M:0;713339a81dd9:37115 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ea18192b38d8424fbdad54a887706b0a, entries=8, sequenceid=319, filesize=5.5 K 2024-12-03T16:10:39,815 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d82aa8c07b3489abba0ff9447455858 as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3d82aa8c07b3489abba0ff9447455858 2024-12-03T16:10:39,822 INFO [M:0;713339a81dd9:37115 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3d82aa8c07b3489abba0ff9447455858 2024-12-03T16:10:39,823 INFO [M:0;713339a81dd9:37115 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3d82aa8c07b3489abba0ff9447455858, entries=33, sequenceid=319, filesize=13.7 K 2024-12-03T16:10:39,824 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/38420afa4d35406bb8f4ced34677044d as hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/38420afa4d35406bb8f4ced34677044d 2024-12-03T16:10:39,831 INFO [M:0;713339a81dd9:37115 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41983/user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/38420afa4d35406bb8f4ced34677044d, entries=1, sequenceid=319, filesize=5.0 K 2024-12-03T16:10:39,833 INFO [M:0;713339a81dd9:37115 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.05 KB/166967, heapSize ~190.82 KB/195400, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1324ms, sequenceid=319, compaction requested=false 2024-12-03T16:10:39,834 INFO [M:0;713339a81dd9:37115 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T16:10:39,835 DEBUG [M:0;713339a81dd9:37115 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733242238509Disabling compacts and flushes for region at 1733242238509Disabling writes for close at 1733242238509Obtaining lock to block concurrent updates at 1733242238510 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733242238510Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166967, getHeapSize=195640, getOffHeapSize=0, getCellsCount=366 at 1733242238510Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733242238511 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733242238511Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733242238527 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733242238527Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733242238941 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733242238959 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733242238959Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733242239373 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733242239389 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733242239389Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1442307d: reopening flushed file at 1733242239804 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68667420: reopening flushed file at 1733242239814 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fa2ed91: reopening flushed file at 1733242239823 (+9 ms)Finished flush of dataSize ~163.05 KB/166967, heapSize ~190.82 KB/195400, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1324ms, sequenceid=319, compaction requested=false at 1733242239833 (+10 ms)Writing region close event to WAL at 1733242239834 (+1 ms)Closed at 1733242239834 2024-12-03T16:10:39,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45443 is added to blk_1073741830_1006 (size=188309) 2024-12-03T16:10:39,838 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/25da468f-5848-ce51-7247-2e24d1508e5c/MasterData/WALs/713339a81dd9,37115,1733242106809/713339a81dd9%2C37115%2C1733242106809.1733242108870 not finished, retry = 0 2024-12-03T16:10:39,940 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T16:10:39,940 INFO [M:0;713339a81dd9:37115 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T16:10:39,940 INFO [M:0;713339a81dd9:37115 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37115 2024-12-03T16:10:39,940 INFO [M:0;713339a81dd9:37115 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T16:10:40,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T16:10:40,042 INFO [M:0;713339a81dd9:37115 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T16:10:40,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37115-0x101793116f70000, quorum=127.0.0.1:57365, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T16:10:40,050 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T16:10:40,052 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T16:10:40,052 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T16:10:40,053 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T16:10:40,053 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/hadoop.log.dir/,STOPPED} 2024-12-03T16:10:40,055 WARN [BP-473686499-172.17.0.3-1733242104029 heartbeating to localhost/127.0.0.1:41983 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T16:10:40,055 WARN [BP-473686499-172.17.0.3-1733242104029 heartbeating to localhost/127.0.0.1:41983 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-473686499-172.17.0.3-1733242104029 (Datanode Uuid 3822169c-e2f0-454f-bdc1-fd5bb72b2ba0) service to localhost/127.0.0.1:41983 2024-12-03T16:10:40,055 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T16:10:40,056 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T16:10:40,057 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/cluster_2b1a8cb5-52d9-e7c9-2269-b219fee5ecf0/data/data1/current/BP-473686499-172.17.0.3-1733242104029 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T16:10:40,057 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/cluster_2b1a8cb5-52d9-e7c9-2269-b219fee5ecf0/data/data2/current/BP-473686499-172.17.0.3-1733242104029 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T16:10:40,057 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T16:10:40,066 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T16:10:40,066 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T16:10:40,066 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T16:10:40,066 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T16:10:40,067 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2015a2a9-e4da-0b51-d5a3-8bd543263e34/hadoop.log.dir/,STOPPED} 2024-12-03T16:10:40,074 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T16:10:40,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down