2024-12-11 17:21:59,821 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-11 17:21:59,851 main DEBUG Took 0.027019 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 17:21:59,851 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 17:21:59,852 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 17:21:59,875 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 17:21:59,876 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,888 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 17:21:59,907 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,910 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,911 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,911 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,912 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,913 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,914 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,914 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,915 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,915 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,916 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,916 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,917 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,918 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,918 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,919 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,919 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,920 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,920 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,921 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,921 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,922 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,922 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 17:21:59,923 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,923 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 17:21:59,925 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 17:21:59,933 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 17:21:59,935 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 17:21:59,936 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 17:21:59,939 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 17:21:59,939 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 17:21:59,955 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 17:21:59,959 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 17:21:59,963 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 17:21:59,965 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 17:21:59,966 main DEBUG createAppenders(={Console}) 2024-12-11 17:21:59,967 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-11 17:21:59,967 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-11 17:21:59,968 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-11 17:21:59,969 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 17:21:59,969 main DEBUG OutputStream closed 2024-12-11 17:21:59,970 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 17:21:59,970 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 17:21:59,971 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-11 17:22:00,043 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 17:22:00,045 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 17:22:00,046 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 17:22:00,047 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 17:22:00,048 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 17:22:00,048 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 17:22:00,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 17:22:00,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 17:22:00,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 17:22:00,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 17:22:00,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 17:22:00,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 17:22:00,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 17:22:00,050 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 17:22:00,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 17:22:00,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 17:22:00,051 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 17:22:00,052 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 17:22:00,054 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 17:22:00,054 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-11 17:22:00,054 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 17:22:00,055 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-11T17:22:00,066 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-11 17:22:00,069 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 17:22:00,069 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T17:22:00,423 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c 2024-12-11T17:22:00,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-11T17:22:00,465 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/cluster_113cb4fb-a821-384d-7ca6-aec806970df0, deleteOnExit=true 2024-12-11T17:22:00,466 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-11T17:22:00,467 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/test.cache.data in system properties and HBase conf 2024-12-11T17:22:00,468 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/hadoop.tmp.dir in system properties and HBase conf 2024-12-11T17:22:00,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/hadoop.log.dir in system properties and HBase conf 2024-12-11T17:22:00,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-11T17:22:00,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-11T17:22:00,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-11T17:22:00,568 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T17:22:00,657 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T17:22:00,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-11T17:22:00,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-11T17:22:00,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-11T17:22:00,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T17:22:00,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-11T17:22:00,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-11T17:22:00,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T17:22:00,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T17:22:00,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-11T17:22:00,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/nfs.dump.dir in system properties and HBase conf 2024-12-11T17:22:00,669 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/java.io.tmpdir in system properties and HBase conf 2024-12-11T17:22:00,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T17:22:00,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-11T17:22:00,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-11T17:22:01,812 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-11T17:22:01,930 INFO [Time-limited test {}] log.Log(170): Logging initialized @3081ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-11T17:22:02,056 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T17:22:02,153 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T17:22:02,183 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T17:22:02,184 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T17:22:02,185 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-11T17:22:02,198 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T17:22:02,200 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/hadoop.log.dir/,AVAILABLE} 2024-12-11T17:22:02,201 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T17:22:02,430 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/java.io.tmpdir/jetty-localhost-38191-hadoop-hdfs-3_4_1-tests_jar-_-any-10586421159728057290/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T17:22:02,443 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:38191} 2024-12-11T17:22:02,443 INFO [Time-limited test {}] server.Server(415): Started @3596ms 2024-12-11T17:22:02,927 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T17:22:02,937 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T17:22:02,938 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T17:22:02,938 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T17:22:02,939 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-11T17:22:02,939 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/hadoop.log.dir/,AVAILABLE} 2024-12-11T17:22:02,940 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T17:22:03,055 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/java.io.tmpdir/jetty-localhost-46371-hadoop-hdfs-3_4_1-tests_jar-_-any-10343221082691417321/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T17:22:03,056 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:46371} 2024-12-11T17:22:03,056 INFO [Time-limited test {}] server.Server(415): Started @4209ms 2024-12-11T17:22:03,113 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T17:22:03,941 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/cluster_113cb4fb-a821-384d-7ca6-aec806970df0/data/data2/current/BP-1757774003-172.17.0.3-1733937721344/current, will proceed with Du for space computation calculation, 2024-12-11T17:22:03,941 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/cluster_113cb4fb-a821-384d-7ca6-aec806970df0/data/data1/current/BP-1757774003-172.17.0.3-1733937721344/current, will proceed with Du for space computation calculation, 2024-12-11T17:22:03,980 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T17:22:04,029 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x40b039c74073b546 with lease ID 0xcf5943a8b333cfb3: Processing first storage report for DS-daf6dfea-6cfc-491f-a6c4-c9b538c6269c from datanode DatanodeRegistration(127.0.0.1:34293, datanodeUuid=40bc0b08-6cc9-448a-bdd3-6dd27ea06d79, infoPort=40695, infoSecurePort=0, ipcPort=34689, storageInfo=lv=-57;cid=testClusterID;nsid=342225494;c=1733937721344) 2024-12-11T17:22:04,030 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x40b039c74073b546 with lease ID 0xcf5943a8b333cfb3: from storage DS-daf6dfea-6cfc-491f-a6c4-c9b538c6269c node DatanodeRegistration(127.0.0.1:34293, datanodeUuid=40bc0b08-6cc9-448a-bdd3-6dd27ea06d79, infoPort=40695, infoSecurePort=0, ipcPort=34689, storageInfo=lv=-57;cid=testClusterID;nsid=342225494;c=1733937721344), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-11T17:22:04,031 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x40b039c74073b546 with lease ID 0xcf5943a8b333cfb3: Processing first storage report for DS-2067154d-a810-44ba-9946-ae623784d093 from datanode DatanodeRegistration(127.0.0.1:34293, datanodeUuid=40bc0b08-6cc9-448a-bdd3-6dd27ea06d79, infoPort=40695, infoSecurePort=0, ipcPort=34689, storageInfo=lv=-57;cid=testClusterID;nsid=342225494;c=1733937721344) 2024-12-11T17:22:04,031 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x40b039c74073b546 with lease ID 0xcf5943a8b333cfb3: from storage DS-2067154d-a810-44ba-9946-ae623784d093 node DatanodeRegistration(127.0.0.1:34293, datanodeUuid=40bc0b08-6cc9-448a-bdd3-6dd27ea06d79, infoPort=40695, infoSecurePort=0, ipcPort=34689, storageInfo=lv=-57;cid=testClusterID;nsid=342225494;c=1733937721344), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T17:22:04,038 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c 2024-12-11T17:22:04,128 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/cluster_113cb4fb-a821-384d-7ca6-aec806970df0/zookeeper_0, clientPort=63961, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/cluster_113cb4fb-a821-384d-7ca6-aec806970df0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/cluster_113cb4fb-a821-384d-7ca6-aec806970df0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T17:22:04,139 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63961 2024-12-11T17:22:04,148 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T17:22:04,152 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T17:22:04,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741825_1001 (size=7) 2024-12-11T17:22:04,828 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 with version=8 2024-12-11T17:22:04,829 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/hbase-staging 2024-12-11T17:22:04,929 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-11T17:22:05,180 INFO [Time-limited test {}] client.ConnectionUtils(128): master/790ba370077b:0 server-side Connection retries=9 2024-12-11T17:22:05,195 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T17:22:05,196 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T17:22:05,201 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T17:22:05,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T17:22:05,201 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T17:22:05,347 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T17:22:05,423 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T17:22:05,435 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T17:22:05,440 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T17:22:05,472 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 10830 (auto-detected) 2024-12-11T17:22:05,474 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-11T17:22:05,507 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38569 2024-12-11T17:22:05,538 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38569 connecting to ZooKeeper ensemble=127.0.0.1:63961 2024-12-11T17:22:05,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385690x0, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T17:22:05,633 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38569-0x10016012ad60000 connected 2024-12-11T17:22:05,709 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T17:22:05,712 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T17:22:05,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T17:22:05,733 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90, hbase.cluster.distributed=false 2024-12-11T17:22:05,766 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T17:22:05,770 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38569 2024-12-11T17:22:05,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38569 2024-12-11T17:22:05,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38569 2024-12-11T17:22:05,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38569 2024-12-11T17:22:05,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38569 2024-12-11T17:22:05,889 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/790ba370077b:0 server-side Connection retries=9 2024-12-11T17:22:05,891 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T17:22:05,891 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T17:22:05,891 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T17:22:05,891 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T17:22:05,891 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T17:22:05,894 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T17:22:05,896 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T17:22:05,897 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39119 2024-12-11T17:22:05,898 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39119 connecting to ZooKeeper ensemble=127.0.0.1:63961 2024-12-11T17:22:05,899 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T17:22:05,902 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T17:22:05,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391190x0, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T17:22:05,917 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39119-0x10016012ad60001 connected 2024-12-11T17:22:05,917 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T17:22:05,921 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T17:22:05,928 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T17:22:05,930 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T17:22:05,936 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T17:22:05,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39119 2024-12-11T17:22:05,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39119 2024-12-11T17:22:05,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39119 2024-12-11T17:22:05,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39119 2024-12-11T17:22:05,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39119 2024-12-11T17:22:05,959 DEBUG [M:0;790ba370077b:38569 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;790ba370077b:38569 2024-12-11T17:22:05,961 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/790ba370077b,38569,1733937724982 2024-12-11T17:22:05,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T17:22:05,978 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T17:22:05,983 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/790ba370077b,38569,1733937724982 2024-12-11T17:22:06,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T17:22:06,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:06,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:06,021 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T17:22:06,033 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/790ba370077b,38569,1733937724982 from backup master directory 2024-12-11T17:22:06,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T17:22:06,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/790ba370077b,38569,1733937724982 2024-12-11T17:22:06,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T17:22:06,050 WARN [master/790ba370077b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T17:22:06,050 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=790ba370077b,38569,1733937724982 2024-12-11T17:22:06,052 INFO [master/790ba370077b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-11T17:22:06,057 INFO [master/790ba370077b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-11T17:22:06,171 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/hbase.id] with ID: 50115fc1-27b4-408b-b685-fbd3fb0ba46a 2024-12-11T17:22:06,171 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/.tmp/hbase.id 2024-12-11T17:22:06,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741826_1002 (size=42) 2024-12-11T17:22:06,599 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/.tmp/hbase.id]:[hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/hbase.id] 2024-12-11T17:22:06,653 INFO [master/790ba370077b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T17:22:06,659 INFO [master/790ba370077b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T17:22:06,680 INFO [master/790ba370077b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-11T17:22:06,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:06,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:06,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741827_1003 (size=196) 2024-12-11T17:22:07,133 INFO [master/790ba370077b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:22:07,138 INFO [master/790ba370077b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T17:22:07,159 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T17:22:07,165 INFO [master/790ba370077b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T17:22:07,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741828_1004 (size=1189) 2024-12-11T17:22:07,263 INFO [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store 2024-12-11T17:22:07,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741829_1005 (size=34) 2024-12-11T17:22:07,325 INFO [master/790ba370077b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-11T17:22:07,329 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:07,331 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T17:22:07,331 INFO [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T17:22:07,331 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T17:22:07,334 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T17:22:07,334 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T17:22:07,334 INFO [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T17:22:07,336 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733937727331Disabling compacts and flushes for region at 1733937727331Disabling writes for close at 1733937727334 (+3 ms)Writing region close event to WAL at 1733937727334Closed at 1733937727334 2024-12-11T17:22:07,340 WARN [master/790ba370077b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/.initializing 2024-12-11T17:22:07,340 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/WALs/790ba370077b,38569,1733937724982 2024-12-11T17:22:07,356 INFO [master/790ba370077b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T17:22:07,380 INFO [master/790ba370077b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=790ba370077b%2C38569%2C1733937724982, suffix=, logDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/WALs/790ba370077b,38569,1733937724982, archiveDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/oldWALs, maxLogs=10 2024-12-11T17:22:07,421 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/WALs/790ba370077b,38569,1733937724982/790ba370077b%2C38569%2C1733937724982.1733937727387, exclude list is [], retry=0 2024-12-11T17:22:07,448 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34293,DS-daf6dfea-6cfc-491f-a6c4-c9b538c6269c,DISK] 2024-12-11T17:22:07,452 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-11T17:22:07,499 INFO [master/790ba370077b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/WALs/790ba370077b,38569,1733937724982/790ba370077b%2C38569%2C1733937724982.1733937727387 2024-12-11T17:22:07,501 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40695:40695)] 2024-12-11T17:22:07,501 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:22:07,502 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:07,507 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,508 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,577 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T17:22:07,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:07,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:07,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T17:22:07,588 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:07,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:22:07,589 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T17:22:07,593 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:07,594 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:22:07,594 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T17:22:07,599 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:07,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:22:07,601 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,605 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,607 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,616 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,617 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,622 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T17:22:07,626 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T17:22:07,632 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:22:07,634 INFO [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64976190, jitterRate=-0.031779319047927856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T17:22:07,640 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733937727523Initializing all the Stores at 1733937727526 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733937727527 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937727528 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937727528Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937727528Cleaning up temporary data from old regions at 1733937727617 (+89 ms)Region opened successfully at 1733937727640 (+23 ms) 2024-12-11T17:22:07,641 INFO [master/790ba370077b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T17:22:07,681 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60ee41b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=790ba370077b/172.17.0.3:0 2024-12-11T17:22:07,716 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T17:22:07,733 INFO [master/790ba370077b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-11T17:22:07,734 INFO [master/790ba370077b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-11T17:22:07,738 INFO [master/790ba370077b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T17:22:07,740 INFO [master/790ba370077b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-11T17:22:07,748 INFO [master/790ba370077b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-11T17:22:07,748 INFO [master/790ba370077b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T17:22:07,790 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T17:22:07,801 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T17:22:07,832 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T17:22:07,836 INFO [master/790ba370077b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T17:22:07,842 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T17:22:07,849 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T17:22:07,852 INFO [master/790ba370077b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T17:22:07,855 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T17:22:07,866 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T17:22:07,867 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T17:22:07,874 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T17:22:07,899 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T17:22:07,947 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T17:22:07,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T17:22:07,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T17:22:07,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:07,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:07,960 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=790ba370077b,38569,1733937724982, sessionid=0x10016012ad60000, setting cluster-up flag (Was=false) 2024-12-11T17:22:07,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:07,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:08,007 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T17:22:08,010 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=790ba370077b,38569,1733937724982 2024-12-11T17:22:08,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:08,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:08,057 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T17:22:08,059 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=790ba370077b,38569,1733937724982 2024-12-11T17:22:08,067 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T17:22:08,148 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(746): ClusterId : 50115fc1-27b4-408b-b685-fbd3fb0ba46a 2024-12-11T17:22:08,149 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T17:22:08,150 DEBUG [RS:0;790ba370077b:39119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T17:22:08,163 INFO [master/790ba370077b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T17:22:08,167 DEBUG [RS:0;790ba370077b:39119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T17:22:08,167 DEBUG [RS:0;790ba370077b:39119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T17:22:08,170 INFO [master/790ba370077b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T17:22:08,176 DEBUG [RS:0;790ba370077b:39119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T17:22:08,176 DEBUG [RS:0;790ba370077b:39119 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581b203e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=790ba370077b/172.17.0.3:0 2024-12-11T17:22:08,177 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 790ba370077b,38569,1733937724982 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T17:22:08,185 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/790ba370077b:0, corePoolSize=5, maxPoolSize=5 2024-12-11T17:22:08,185 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/790ba370077b:0, corePoolSize=5, maxPoolSize=5 2024-12-11T17:22:08,185 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/790ba370077b:0, corePoolSize=5, maxPoolSize=5 2024-12-11T17:22:08,185 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/790ba370077b:0, corePoolSize=5, maxPoolSize=5 2024-12-11T17:22:08,186 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/790ba370077b:0, corePoolSize=10, maxPoolSize=10 2024-12-11T17:22:08,186 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,186 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/790ba370077b:0, corePoolSize=2, maxPoolSize=2 2024-12-11T17:22:08,187 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,193 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;790ba370077b:39119 2024-12-11T17:22:08,193 INFO [master/790ba370077b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733937758193 2024-12-11T17:22:08,195 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T17:22:08,196 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T17:22:08,196 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T17:22:08,197 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T17:22:08,197 INFO [RS:0;790ba370077b:39119 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T17:22:08,197 INFO [RS:0;790ba370077b:39119 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T17:22:08,197 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T17:22:08,201 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(2659): reportForDuty to master=790ba370077b,38569,1733937724982 with port=39119, startcode=1733937725853 2024-12-11T17:22:08,201 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T17:22:08,201 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T17:22:08,202 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T17:22:08,202 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T17:22:08,204 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:08,203 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,208 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T17:22:08,213 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T17:22:08,215 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T17:22:08,215 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T17:22:08,218 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T17:22:08,218 INFO [master/790ba370077b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T17:22:08,221 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/790ba370077b:0:becomeActiveMaster-HFileCleaner.large.0-1733937728220,5,FailOnTimeoutGroup] 2024-12-11T17:22:08,221 DEBUG [RS:0;790ba370077b:39119 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T17:22:08,224 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/790ba370077b:0:becomeActiveMaster-HFileCleaner.small.0-1733937728221,5,FailOnTimeoutGroup] 2024-12-11T17:22:08,225 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,225 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T17:22:08,227 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,227 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741831_1007 (size=1321) 2024-12-11T17:22:08,235 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T17:22:08,236 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:22:08,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741832_1008 (size=32) 2024-12-11T17:22:08,268 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37981, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T17:22:08,274 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38569 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 790ba370077b,39119,1733937725853 2024-12-11T17:22:08,277 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38569 {}] master.ServerManager(517): Registering regionserver=790ba370077b,39119,1733937725853 2024-12-11T17:22:08,291 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:22:08,291 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34667 2024-12-11T17:22:08,292 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T17:22:08,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T17:22:08,300 DEBUG [RS:0;790ba370077b:39119 {}] zookeeper.ZKUtil(111): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/790ba370077b,39119,1733937725853 2024-12-11T17:22:08,300 WARN [RS:0;790ba370077b:39119 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T17:22:08,300 INFO [RS:0;790ba370077b:39119 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T17:22:08,300 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/WALs/790ba370077b,39119,1733937725853 2024-12-11T17:22:08,302 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [790ba370077b,39119,1733937725853] 2024-12-11T17:22:08,339 INFO [RS:0;790ba370077b:39119 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T17:22:08,357 INFO [RS:0;790ba370077b:39119 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T17:22:08,364 INFO [RS:0;790ba370077b:39119 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T17:22:08,364 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,365 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T17:22:08,371 INFO [RS:0;790ba370077b:39119 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T17:22:08,372 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,372 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,372 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,372 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,373 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,373 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,373 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/790ba370077b:0, corePoolSize=2, maxPoolSize=2 2024-12-11T17:22:08,373 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,373 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,373 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,373 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,374 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,374 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/790ba370077b:0, corePoolSize=1, maxPoolSize=1 2024-12-11T17:22:08,374 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/790ba370077b:0, corePoolSize=3, maxPoolSize=3 2024-12-11T17:22:08,374 DEBUG [RS:0;790ba370077b:39119 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/790ba370077b:0, corePoolSize=3, maxPoolSize=3 2024-12-11T17:22:08,375 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,375 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,375 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,375 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,375 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,376 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=790ba370077b,39119,1733937725853-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T17:22:08,393 INFO [RS:0;790ba370077b:39119 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-11T17:22:08,395 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=790ba370077b,39119,1733937725853-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,395 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,396 INFO [RS:0;790ba370077b:39119 {}] regionserver.Replication(171): 790ba370077b,39119,1733937725853 started 2024-12-11T17:22:08,419 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:08,419 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1482): Serving as 790ba370077b,39119,1733937725853, RpcServer on 790ba370077b/172.17.0.3:39119, sessionid=0x10016012ad60001 2024-12-11T17:22:08,420 DEBUG [RS:0;790ba370077b:39119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T17:22:08,420 DEBUG [RS:0;790ba370077b:39119 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 790ba370077b,39119,1733937725853 2024-12-11T17:22:08,421 DEBUG [RS:0;790ba370077b:39119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '790ba370077b,39119,1733937725853' 2024-12-11T17:22:08,421 DEBUG [RS:0;790ba370077b:39119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T17:22:08,422 DEBUG [RS:0;790ba370077b:39119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T17:22:08,423 DEBUG [RS:0;790ba370077b:39119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T17:22:08,423 DEBUG [RS:0;790ba370077b:39119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T17:22:08,423 DEBUG [RS:0;790ba370077b:39119 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 790ba370077b,39119,1733937725853 2024-12-11T17:22:08,423 DEBUG [RS:0;790ba370077b:39119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '790ba370077b,39119,1733937725853' 2024-12-11T17:22:08,423 DEBUG [RS:0;790ba370077b:39119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T17:22:08,424 DEBUG [RS:0;790ba370077b:39119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T17:22:08,425 DEBUG [RS:0;790ba370077b:39119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T17:22:08,425 INFO [RS:0;790ba370077b:39119 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T17:22:08,425 INFO [RS:0;790ba370077b:39119 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T17:22:08,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:08,670 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T17:22:08,674 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T17:22:08,674 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:08,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:08,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T17:22:08,681 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T17:22:08,681 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:08,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:08,683 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T17:22:08,688 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T17:22:08,688 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:08,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:08,690 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T17:22:08,693 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T17:22:08,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:08,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:08,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T17:22:08,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740 2024-12-11T17:22:08,700 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740 2024-12-11T17:22:08,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T17:22:08,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T17:22:08,707 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T17:22:08,710 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T17:22:08,726 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:22:08,728 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66243582, jitterRate=-0.012893706560134888}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T17:22:08,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733937728667Initializing all the Stores at 1733937728669 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733937728669Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733937728670 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937728670Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733937728670Cleaning up temporary data from old regions at 1733937728704 (+34 ms)Region opened successfully at 1733937728732 (+28 ms) 2024-12-11T17:22:08,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T17:22:08,733 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T17:22:08,733 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T17:22:08,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T17:22:08,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T17:22:08,739 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T17:22:08,740 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733937728733Disabling compacts and flushes for region at 1733937728733Disabling writes for close at 1733937728737 (+4 ms)Writing region close event to WAL at 1733937728738 (+1 ms)Closed at 1733937728739 (+1 ms) 2024-12-11T17:22:08,746 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T17:22:08,747 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T17:22:08,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T17:22:08,771 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T17:22:08,778 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T17:22:08,929 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:09,930 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:10,931 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:11,932 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:12,933 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:13,934 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:14,934 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:15,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-11T17:22:15,417 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-11T17:22:15,935 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:16,936 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:17,936 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:18,937 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:19,938 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:20,939 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:21,939 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:22,940 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:23,941 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:24,941 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:25,942 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:26,943 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:27,943 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:28,944 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:29,945 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:30,946 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:31,947 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:32,948 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:33,948 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:34,033 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T17:22:34,949 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:35,950 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:36,950 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:37,951 WARN [790ba370077b:38569 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-11T17:22:38,434 INFO [RS:0;790ba370077b:39119 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T17:22:38,439 INFO [RS:0;790ba370077b:39119 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=790ba370077b%2C39119%2C1733937725853, suffix=, logDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/WALs/790ba370077b,39119,1733937725853, archiveDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/oldWALs, maxLogs=32 2024-12-11T17:22:38,454 DEBUG [RS:0;790ba370077b:39119 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/WALs/790ba370077b,39119,1733937725853/790ba370077b%2C39119%2C1733937725853.1733937758441, exclude list is [], retry=0 2024-12-11T17:22:38,459 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34293,DS-daf6dfea-6cfc-491f-a6c4-c9b538c6269c,DISK] 2024-12-11T17:22:38,463 INFO [RS:0;790ba370077b:39119 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/WALs/790ba370077b,39119,1733937725853/790ba370077b%2C39119%2C1733937725853.1733937758441 2024-12-11T17:22:38,464 DEBUG [RS:0;790ba370077b:39119 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40695:40695)] 2024-12-11T17:22:38,704 DEBUG [790ba370077b:38569 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-11T17:22:38,717 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:22:38,724 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 790ba370077b,39119,1733937725853, state=OPENING 2024-12-11T17:22:38,762 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T17:22:38,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:38,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:22:38,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T17:22:38,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T17:22:38,776 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T17:22:38,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:22:38,958 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T17:22:38,962 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45783, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T17:22:38,974 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T17:22:38,974 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T17:22:38,975 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T17:22:38,979 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=790ba370077b%2C39119%2C1733937725853.meta, suffix=.meta, logDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/WALs/790ba370077b,39119,1733937725853, archiveDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/oldWALs, maxLogs=32 2024-12-11T17:22:38,993 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/WALs/790ba370077b,39119,1733937725853/790ba370077b%2C39119%2C1733937725853.meta.1733937758980.meta, exclude list is [], retry=0 2024-12-11T17:22:38,997 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34293,DS-daf6dfea-6cfc-491f-a6c4-c9b538c6269c,DISK] 2024-12-11T17:22:39,000 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/WALs/790ba370077b,39119,1733937725853/790ba370077b%2C39119%2C1733937725853.meta.1733937758980.meta 2024-12-11T17:22:39,001 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40695:40695)] 2024-12-11T17:22:39,002 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:22:39,003 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T17:22:39,006 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T17:22:39,011 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T17:22:39,015 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T17:22:39,016 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:39,016 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T17:22:39,016 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T17:22:39,020 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T17:22:39,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T17:22:39,022 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:39,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:39,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T17:22:39,024 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T17:22:39,024 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:39,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:39,026 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T17:22:39,027 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T17:22:39,027 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:39,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:39,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T17:22:39,030 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T17:22:39,030 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:39,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T17:22:39,031 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T17:22:39,033 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740 2024-12-11T17:22:39,036 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740 2024-12-11T17:22:39,038 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T17:22:39,039 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T17:22:39,040 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T17:22:39,043 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T17:22:39,045 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60935649, jitterRate=-0.09198807179927826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T17:22:39,045 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T17:22:39,047 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733937759017Writing region info on filesystem at 1733937759017Initializing all the Stores at 1733937759019 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733937759019Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733937759020 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937759020Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733937759020Cleaning up temporary data from old regions at 1733937759039 (+19 ms)Running coprocessor post-open hooks at 1733937759045 (+6 ms)Region opened successfully at 1733937759047 (+2 ms) 2024-12-11T17:22:39,055 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733937758949 2024-12-11T17:22:39,067 DEBUG [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T17:22:39,068 INFO [RS_OPEN_META-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T17:22:39,070 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:22:39,072 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 790ba370077b,39119,1733937725853, state=OPEN 2024-12-11T17:22:39,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T17:22:39,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T17:22:39,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T17:22:39,119 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T17:22:39,120 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=790ba370077b,39119,1733937725853 2024-12-11T17:22:39,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T17:22:39,126 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=790ba370077b,39119,1733937725853 in 342 msec 2024-12-11T17:22:39,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T17:22:39,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.3730 sec 2024-12-11T17:22:39,138 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T17:22:39,139 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T17:22:39,159 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T17:22:39,160 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=790ba370077b,39119,1733937725853, seqNum=-1] 2024-12-11T17:22:39,180 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T17:22:39,182 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35031, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T17:22:39,202 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.1010 sec 2024-12-11T17:22:39,203 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733937759202, completionTime=-1 2024-12-11T17:22:39,205 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-11T17:22:39,205 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T17:22:39,234 INFO [master/790ba370077b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-11T17:22:39,234 INFO [master/790ba370077b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733937819234 2024-12-11T17:22:39,235 INFO [master/790ba370077b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733937879235 2024-12-11T17:22:39,235 INFO [master/790ba370077b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-11T17:22:39,236 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=790ba370077b,38569,1733937724982-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:39,237 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=790ba370077b,38569,1733937724982-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:39,237 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=790ba370077b,38569,1733937724982-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:39,238 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-790ba370077b:38569, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:39,238 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:39,239 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:39,245 DEBUG [master/790ba370077b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T17:22:39,268 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.218sec 2024-12-11T17:22:39,269 INFO [master/790ba370077b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T17:22:39,270 INFO [master/790ba370077b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T17:22:39,271 INFO [master/790ba370077b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T17:22:39,272 INFO [master/790ba370077b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T17:22:39,272 INFO [master/790ba370077b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T17:22:39,273 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=790ba370077b,38569,1733937724982-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T17:22:39,274 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=790ba370077b,38569,1733937724982-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T17:22:39,283 DEBUG [master/790ba370077b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T17:22:39,284 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T17:22:39,284 INFO [master/790ba370077b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=790ba370077b,38569,1733937724982-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T17:22:39,284 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77804624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T17:22:39,287 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-11T17:22:39,287 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-11T17:22:39,290 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 790ba370077b,38569,-1 for getting cluster id 2024-12-11T17:22:39,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T17:22:39,303 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '50115fc1-27b4-408b-b685-fbd3fb0ba46a' 2024-12-11T17:22:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T17:22:39,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "50115fc1-27b4-408b-b685-fbd3fb0ba46a" 2024-12-11T17:22:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71fe4d5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T17:22:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [790ba370077b,38569,-1] 2024-12-11T17:22:39,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T17:22:39,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:22:39,323 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T17:22:39,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fad6539, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T17:22:39,328 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T17:22:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=790ba370077b,39119,1733937725853, seqNum=-1] 2024-12-11T17:22:39,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T17:22:39,338 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T17:22:39,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=790ba370077b,38569,1733937724982 2024-12-11T17:22:39,384 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68214a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T17:22:39,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 790ba370077b,38569,-1 for getting cluster id 2024-12-11T17:22:39,385 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T17:22:39,387 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '50115fc1-27b4-408b-b685-fbd3fb0ba46a' 2024-12-11T17:22:39,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T17:22:39,387 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "50115fc1-27b4-408b-b685-fbd3fb0ba46a" 2024-12-11T17:22:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58c4d5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T17:22:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [790ba370077b,38569,-1] 2024-12-11T17:22:39,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T17:22:39,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:22:39,390 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T17:22:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666749f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T17:22:39,410 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=4375 2024-12-11T17:22:39,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T17:22:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 790ba370077b,38569,1733937724982 2024-12-11T17:22:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2fad16ae 2024-12-11T17:22:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T17:22:39,429 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T17:22:39,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:22:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-11T17:22:39,450 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:22:39,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-11T17:22:39,453 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:39,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:22:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T17:22:39,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741835_1011 (size=428) 2024-12-11T17:22:39,871 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6aa8aadf7fa2b36d8b0c503f2cd10a64, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:22:39,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741836_1012 (size=79) 2024-12-11T17:22:40,288 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:40,289 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 6aa8aadf7fa2b36d8b0c503f2cd10a64, disabling compactions & flushes 2024-12-11T17:22:40,289 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:22:40,290 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:22:40,290 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. after waiting 0 ms 2024-12-11T17:22:40,290 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:22:40,291 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:22:40,291 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6aa8aadf7fa2b36d8b0c503f2cd10a64: Waiting for close lock at 1733937760289Disabling compacts and flushes for region at 1733937760289Disabling writes for close at 1733937760290 (+1 ms)Writing region close event to WAL at 1733937760290Closed at 1733937760290 2024-12-11T17:22:40,295 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:22:40,300 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733937760295"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937760295"}]},"ts":"1733937760295"} 2024-12-11T17:22:40,305 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:22:40,307 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:22:40,310 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937760307"}]},"ts":"1733937760307"} 2024-12-11T17:22:40,314 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-11T17:22:40,316 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=6aa8aadf7fa2b36d8b0c503f2cd10a64, ASSIGN}] 2024-12-11T17:22:40,319 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=6aa8aadf7fa2b36d8b0c503f2cd10a64, ASSIGN 2024-12-11T17:22:40,320 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=6aa8aadf7fa2b36d8b0c503f2cd10a64, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:22:40,472 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6aa8aadf7fa2b36d8b0c503f2cd10a64, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:22:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T17:22:40,477 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=6aa8aadf7fa2b36d8b0c503f2cd10a64, ASSIGN because future has completed 2024-12-11T17:22:40,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6aa8aadf7fa2b36d8b0c503f2cd10a64, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:22:40,643 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:22:40,644 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6aa8aadf7fa2b36d8b0c503f2cd10a64, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:22:40,644 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,645 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:40,645 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,645 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,648 INFO [StoreOpener-6aa8aadf7fa2b36d8b0c503f2cd10a64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,651 INFO [StoreOpener-6aa8aadf7fa2b36d8b0c503f2cd10a64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6aa8aadf7fa2b36d8b0c503f2cd10a64 columnFamilyName testFamily 2024-12-11T17:22:40,651 DEBUG [StoreOpener-6aa8aadf7fa2b36d8b0c503f2cd10a64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:40,653 INFO [StoreOpener-6aa8aadf7fa2b36d8b0c503f2cd10a64-1 {}] regionserver.HStore(327): Store=6aa8aadf7fa2b36d8b0c503f2cd10a64/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:22:40,653 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,655 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,656 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,657 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,657 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,660 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,664 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:22:40,665 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6aa8aadf7fa2b36d8b0c503f2cd10a64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72994698, jitterRate=0.08770576119422913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:22:40,665 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:22:40,667 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6aa8aadf7fa2b36d8b0c503f2cd10a64: Running coprocessor pre-open hook at 1733937760645Writing region info on filesystem at 1733937760645Initializing all the Stores at 1733937760647 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937760648 (+1 ms)Cleaning up temporary data from old regions at 1733937760657 (+9 ms)Running coprocessor post-open hooks at 1733937760665 (+8 ms)Region opened successfully at 1733937760666 (+1 ms) 2024-12-11T17:22:40,669 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64., pid=6, masterSystemTime=1733937760634 2024-12-11T17:22:40,673 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:22:40,673 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:22:40,674 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6aa8aadf7fa2b36d8b0c503f2cd10a64, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:22:40,678 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6aa8aadf7fa2b36d8b0c503f2cd10a64, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:22:40,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-11T17:22:40,685 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6aa8aadf7fa2b36d8b0c503f2cd10a64, server=790ba370077b,39119,1733937725853 in 202 msec 2024-12-11T17:22:40,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-11T17:22:40,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=6aa8aadf7fa2b36d8b0c503f2cd10a64, ASSIGN in 369 msec 2024-12-11T17:22:40,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:22:40,691 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937760691"}]},"ts":"1733937760691"} 2024-12-11T17:22:40,694 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-11T17:22:40,696 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:22:40,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2550 sec 2024-12-11T17:22:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T17:22:42,495 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-11T17:22:42,495 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-11T17:22:42,497 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:22:42,506 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-11T17:22:42,506 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:22:42,507 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-11T17:22:42,509 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T17:22:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=790ba370077b,39119,1733937725853, seqNum=-1] 2024-12-11T17:22:42,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T17:22:42,515 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T17:22:42,523 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:22:42,542 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:22:42,547 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-11T17:22:42,553 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-11T17:22:42,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-11T17:22:42,561 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-11T17:22:42,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6870863297436975096 2024-12-11T17:22:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:51980 deadline: 1733937765561, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T17:22:42,596 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-11T17:22:42,597 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-11T17:22:42,599 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:22:42,600 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:22:42,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6870863297436975094 2024-12-11T17:22:42,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:51980 deadline: 1733937765600, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T17:22:42,605 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:22:42,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6870863297436975093 2024-12-11T17:22:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:51980 deadline: 1733937765605, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T17:22:42,611 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:22:42,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6870863297436975092 2024-12-11T17:22:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:51980 deadline: 1733937765611, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T17:22:42,616 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:22:42,627 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RS_OPEN_REGION-regionserver/790ba370077b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-538874806_22 at /127.0.0.1:41794 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=630 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=4369 (was 4375) 2024-12-11T17:22:42,635 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=630, ProcessCount=11, AvailableMemoryMB=4369 2024-12-11T17:22:42,639 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:22:42,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-11T17:22:42,643 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:22:42,643 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:42,644 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-11T17:22:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T17:22:42,646 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:22:42,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741837_1013 (size=414) 2024-12-11T17:22:43,059 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1be97a5d392c877b9bc9b2732b8cfea7, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:22:43,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741838_1014 (size=65) 2024-12-11T17:22:43,482 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:43,482 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 1be97a5d392c877b9bc9b2732b8cfea7, disabling compactions & flushes 2024-12-11T17:22:43,482 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:22:43,482 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:22:43,482 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. after waiting 0 ms 2024-12-11T17:22:43,482 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:22:43,482 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:22:43,482 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1be97a5d392c877b9bc9b2732b8cfea7: Waiting for close lock at 1733937763482Disabling compacts and flushes for region at 1733937763482Disabling writes for close at 1733937763482Writing region close event to WAL at 1733937763482Closed at 1733937763482 2024-12-11T17:22:43,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:22:43,486 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733937763486"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937763486"}]},"ts":"1733937763486"} 2024-12-11T17:22:43,491 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:22:43,494 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:22:43,494 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937763494"}]},"ts":"1733937763494"} 2024-12-11T17:22:43,498 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-11T17:22:43,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=1be97a5d392c877b9bc9b2732b8cfea7, ASSIGN}] 2024-12-11T17:22:43,501 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=1be97a5d392c877b9bc9b2732b8cfea7, ASSIGN 2024-12-11T17:22:43,503 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=1be97a5d392c877b9bc9b2732b8cfea7, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:22:43,655 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1be97a5d392c877b9bc9b2732b8cfea7, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:22:43,663 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=1be97a5d392c877b9bc9b2732b8cfea7, ASSIGN because future has completed 2024-12-11T17:22:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T17:22:43,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1be97a5d392c877b9bc9b2732b8cfea7, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:22:43,826 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:22:43,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1be97a5d392c877b9bc9b2732b8cfea7, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:22:43,826 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:43,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,827 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,830 INFO [StoreOpener-1be97a5d392c877b9bc9b2732b8cfea7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,836 INFO [StoreOpener-1be97a5d392c877b9bc9b2732b8cfea7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1be97a5d392c877b9bc9b2732b8cfea7 columnFamilyName testFamily 2024-12-11T17:22:43,836 DEBUG [StoreOpener-1be97a5d392c877b9bc9b2732b8cfea7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:43,837 INFO [StoreOpener-1be97a5d392c877b9bc9b2732b8cfea7-1 {}] regionserver.HStore(327): Store=1be97a5d392c877b9bc9b2732b8cfea7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:22:43,837 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,838 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,839 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,839 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,840 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,842 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,846 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:22:43,847 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 1be97a5d392c877b9bc9b2732b8cfea7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69344577, jitterRate=0.033314719796180725}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:22:43,847 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:22:43,848 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1be97a5d392c877b9bc9b2732b8cfea7: Running coprocessor pre-open hook at 1733937763827Writing region info on filesystem at 1733937763827Initializing all the Stores at 1733937763830 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937763830Cleaning up temporary data from old regions at 1733937763840 (+10 ms)Running coprocessor post-open hooks at 1733937763847 (+7 ms)Region opened successfully at 1733937763848 (+1 ms) 2024-12-11T17:22:43,849 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7., pid=9, masterSystemTime=1733937763818 2024-12-11T17:22:43,853 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:22:43,853 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:22:43,854 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1be97a5d392c877b9bc9b2732b8cfea7, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:22:43,858 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1be97a5d392c877b9bc9b2732b8cfea7, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:22:43,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-11T17:22:43,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1be97a5d392c877b9bc9b2732b8cfea7, server=790ba370077b,39119,1733937725853 in 196 msec 2024-12-11T17:22:43,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-11T17:22:43,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=1be97a5d392c877b9bc9b2732b8cfea7, ASSIGN in 365 msec 2024-12-11T17:22:43,869 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:22:43,869 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937763869"}]},"ts":"1733937763869"} 2024-12-11T17:22:43,872 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-11T17:22:43,874 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:22:43,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2350 sec 2024-12-11T17:22:44,551 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T17:22:44,582 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-11T17:22:44,585 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T17:22:44,586 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-11T17:22:45,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-11T17:22:45,417 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-11T17:22:45,420 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T17:22:45,420 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-11T17:22:45,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-11T17:22:45,422 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T17:22:45,425 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-11T17:22:45,425 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-11T17:22:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T17:22:45,686 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-11T17:22:45,686 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-11T17:22:45,686 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:22:45,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-11T17:22:45,696 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:22:45,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-11T17:22:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:22:45,717 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:22:45,717 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-11T17:22:45,721 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-11T17:22:45,723 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-11T17:22:51,734 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7., hostname=790ba370077b,39119,1733937725853, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7., hostname=790ba370077b,39119,1733937725853, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=790ba370077b:39119 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-12-11T17:22:51,735 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7., hostname=790ba370077b,39119,1733937725853, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6011ms, rpcTimeout=5999ms 2024-12-11T17:22:51,735 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7., hostname=790ba370077b,39119,1733937725853, seqNum=2 from cache 2024-12-11T17:22:51,743 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=572 (was 630), ProcessCount=11 (was 11), AvailableMemoryMB=4940 (was 4369) - AvailableMemoryMB LEAK? - 2024-12-11T17:22:51,751 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=572, ProcessCount=11, AvailableMemoryMB=4940 2024-12-11T17:22:51,753 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:22:51,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-11T17:22:51,757 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:22:51,758 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:51,758 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-11T17:22:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-11T17:22:51,759 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:22:51,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741839_1015 (size=412) 2024-12-11T17:22:52,171 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30d92a4bafdc46c54d1ef5b985214c88, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:22:52,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741840_1016 (size=63) 2024-12-11T17:22:52,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-12-11T17:22:52,583 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:52,584 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 30d92a4bafdc46c54d1ef5b985214c88, disabling compactions & flushes 2024-12-11T17:22:52,584 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:22:52,584 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:22:52,584 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. after waiting 0 ms 2024-12-11T17:22:52,584 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:22:52,584 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:22:52,584 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30d92a4bafdc46c54d1ef5b985214c88: Waiting for close lock at 1733937772584Disabling compacts and flushes for region at 1733937772584Disabling writes for close at 1733937772584Writing region close event to WAL at 1733937772584Closed at 1733937772584 2024-12-11T17:22:52,586 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:22:52,586 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733937772586"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937772586"}]},"ts":"1733937772586"} 2024-12-11T17:22:52,589 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:22:52,591 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:22:52,591 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937772591"}]},"ts":"1733937772591"} 2024-12-11T17:22:52,594 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-11T17:22:52,594 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=30d92a4bafdc46c54d1ef5b985214c88, ASSIGN}] 2024-12-11T17:22:52,597 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=30d92a4bafdc46c54d1ef5b985214c88, ASSIGN 2024-12-11T17:22:52,598 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=30d92a4bafdc46c54d1ef5b985214c88, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:22:52,749 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=30d92a4bafdc46c54d1ef5b985214c88, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:22:52,753 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=30d92a4bafdc46c54d1ef5b985214c88, ASSIGN because future has completed 2024-12-11T17:22:52,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30d92a4bafdc46c54d1ef5b985214c88, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:22:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-11T17:22:52,913 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:22:52,913 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 30d92a4bafdc46c54d1ef5b985214c88, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:22:52,913 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,913 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:22:52,914 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,914 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,916 INFO [StoreOpener-30d92a4bafdc46c54d1ef5b985214c88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,918 INFO [StoreOpener-30d92a4bafdc46c54d1ef5b985214c88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30d92a4bafdc46c54d1ef5b985214c88 columnFamilyName testFamily 2024-12-11T17:22:52,918 DEBUG [StoreOpener-30d92a4bafdc46c54d1ef5b985214c88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:22:52,919 INFO [StoreOpener-30d92a4bafdc46c54d1ef5b985214c88-1 {}] regionserver.HStore(327): Store=30d92a4bafdc46c54d1ef5b985214c88/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:22:52,919 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,920 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,921 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,921 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,921 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,924 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,927 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:22:52,928 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 30d92a4bafdc46c54d1ef5b985214c88; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71721889, jitterRate=0.06873942911624908}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:22:52,928 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:22:52,929 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 30d92a4bafdc46c54d1ef5b985214c88: Running coprocessor pre-open hook at 1733937772914Writing region info on filesystem at 1733937772914Initializing all the Stores at 1733937772915 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937772915Cleaning up temporary data from old regions at 1733937772921 (+6 ms)Running coprocessor post-open hooks at 1733937772928 (+7 ms)Region opened successfully at 1733937772929 (+1 ms) 2024-12-11T17:22:52,930 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88., pid=12, masterSystemTime=1733937772907 2024-12-11T17:22:52,934 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:22:52,934 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:22:52,935 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=30d92a4bafdc46c54d1ef5b985214c88, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:22:52,938 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30d92a4bafdc46c54d1ef5b985214c88, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:22:52,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-11T17:22:52,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 30d92a4bafdc46c54d1ef5b985214c88, server=790ba370077b,39119,1733937725853 in 186 msec 2024-12-11T17:22:52,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-11T17:22:52,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=30d92a4bafdc46c54d1ef5b985214c88, ASSIGN in 350 msec 2024-12-11T17:22:52,949 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:22:52,949 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937772949"}]},"ts":"1733937772949"} 2024-12-11T17:22:52,952 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-11T17:22:52,953 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:22:52,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2000 sec 2024-12-11T17:22:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-11T17:22:54,784 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-11T17:22:54,784 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-11T17:22:54,784 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:22:54,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-11T17:22:54,789 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:22:54,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-11T17:22:54,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:22:54,806 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:22:54,806 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-11T17:22:54,809 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-11T17:22:54,811 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-11T17:22:54,813 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-11T17:22:54,814 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-11T17:22:55,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-11T17:22:55,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T17:22:56,588 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-11T17:22:58,433 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T17:23:00,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39119 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975083' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-11T17:23:00,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39119 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:51980 deadline: 1733937786823, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975083' does not exist 2024-12-11T17:23:00,827 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 790ba370077b,39119,1733937725853 for closing scanner id = -6870863297436975083 for 30d92a4bafdc46c54d1ef5b985214c88 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975083' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T17:23:00,829 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=539 (was 572), ProcessCount=11 (was 11), AvailableMemoryMB=4622 (was 4940) 2024-12-11T17:23:00,837 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=539, ProcessCount=11, AvailableMemoryMB=4622 2024-12-11T17:23:00,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:23:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-11T17:23:00,843 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:23:00,843 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:00,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-11T17:23:00,845 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:23:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T17:23:00,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741841_1017 (size=423) 2024-12-11T17:23:01,255 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88c575eda49795e049781ca96698701c, NAME => 'testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:23:01,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741842_1018 (size=74) 2024-12-11T17:23:01,265 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:01,265 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 88c575eda49795e049781ca96698701c, disabling compactions & flushes 2024-12-11T17:23:01,265 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:01,265 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:01,265 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. after waiting 0 ms 2024-12-11T17:23:01,265 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:01,265 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:01,265 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88c575eda49795e049781ca96698701c: Waiting for close lock at 1733937781265Disabling compacts and flushes for region at 1733937781265Disabling writes for close at 1733937781265Writing region close event to WAL at 1733937781265Closed at 1733937781265 2024-12-11T17:23:01,267 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:23:01,268 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733937781267"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937781267"}]},"ts":"1733937781267"} 2024-12-11T17:23:01,271 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:23:01,272 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:23:01,273 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937781272"}]},"ts":"1733937781272"} 2024-12-11T17:23:01,276 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-11T17:23:01,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=88c575eda49795e049781ca96698701c, ASSIGN}] 2024-12-11T17:23:01,278 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=88c575eda49795e049781ca96698701c, ASSIGN 2024-12-11T17:23:01,280 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=88c575eda49795e049781ca96698701c, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:23:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-12-11T17:23:01,431 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=88c575eda49795e049781ca96698701c, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:01,434 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=88c575eda49795e049781ca96698701c, ASSIGN because future has completed 2024-12-11T17:23:01,435 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88c575eda49795e049781ca96698701c, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:23:01,592 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:01,592 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 88c575eda49795e049781ca96698701c, NAME => 'testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:23:01,593 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,593 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:01,593 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,593 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,595 INFO [StoreOpener-88c575eda49795e049781ca96698701c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,598 INFO [StoreOpener-88c575eda49795e049781ca96698701c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88c575eda49795e049781ca96698701c columnFamilyName testFamily 2024-12-11T17:23:01,598 DEBUG [StoreOpener-88c575eda49795e049781ca96698701c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:01,599 INFO [StoreOpener-88c575eda49795e049781ca96698701c-1 {}] regionserver.HStore(327): Store=88c575eda49795e049781ca96698701c/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:23:01,599 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,600 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,601 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,602 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,602 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,604 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,608 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:23:01,608 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 88c575eda49795e049781ca96698701c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69446083, jitterRate=0.034827277064323425}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:23:01,608 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88c575eda49795e049781ca96698701c 2024-12-11T17:23:01,609 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 88c575eda49795e049781ca96698701c: Running coprocessor pre-open hook at 1733937781593Writing region info on filesystem at 1733937781593Initializing all the Stores at 1733937781595 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937781595Cleaning up temporary data from old regions at 1733937781602 (+7 ms)Running coprocessor post-open hooks at 1733937781608 (+6 ms)Region opened successfully at 1733937781609 (+1 ms) 2024-12-11T17:23:01,610 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c., pid=15, masterSystemTime=1733937781587 2024-12-11T17:23:01,613 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:01,613 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:01,614 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=88c575eda49795e049781ca96698701c, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:01,617 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88c575eda49795e049781ca96698701c, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:23:01,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-11T17:23:01,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 88c575eda49795e049781ca96698701c, server=790ba370077b,39119,1733937725853 in 184 msec 2024-12-11T17:23:01,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-11T17:23:01,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=88c575eda49795e049781ca96698701c, ASSIGN in 347 msec 2024-12-11T17:23:01,627 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:23:01,627 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937781627"}]},"ts":"1733937781627"} 2024-12-11T17:23:01,630 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-11T17:23:01,632 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:23:01,635 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 792 msec 2024-12-11T17:23:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T17:23:01,854 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-11T17:23:01,854 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-11T17:23:01,854 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:01,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-11T17:23:01,859 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:01,859 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-11T17:23:01,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:01,873 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:23:01,873 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-11T17:23:01,876 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-11T17:23:01,879 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-11T17:23:01,880 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-11T17:23:01,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6870863297436975079 2024-12-11T17:23:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:51980 deadline: 1733937784879, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T17:23:01,884 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-11T17:23:01,885 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-11T17:23:01,886 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:23:01,887 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:23:01,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6870863297436975077 2024-12-11T17:23:01,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:51980 deadline: 1733937784887, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T17:23:01,892 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:23:01,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6870863297436975076 2024-12-11T17:23:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:51980 deadline: 1733937784891, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T17:23:01,896 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:23:01,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6870863297436975075 2024-12-11T17:23:01,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.3:51980 deadline: 1733937784895, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-11T17:23:01,900 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-11T17:23:01,910 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=228 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-538874806_22 at /127.0.0.1:54908 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=539 (was 539), ProcessCount=11 (was 11), AvailableMemoryMB=4542 (was 4622) 2024-12-11T17:23:01,919 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=228, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=539, ProcessCount=11, AvailableMemoryMB=4541 2024-12-11T17:23:01,921 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:23:01,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-11T17:23:01,925 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:23:01,925 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:01,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-11T17:23:01,926 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:23:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-11T17:23:01,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741843_1019 (size=418) 2024-12-11T17:23:02,340 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb0260ace7c4b7e117e26b221bf6a0d2, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:23:02,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741844_1020 (size=69) 2024-12-11T17:23:02,590 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-11T17:23:02,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:02,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing cb0260ace7c4b7e117e26b221bf6a0d2, disabling compactions & flushes 2024-12-11T17:23:02,753 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:02,753 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:02,754 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. after waiting 0 ms 2024-12-11T17:23:02,754 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:02,754 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:02,754 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb0260ace7c4b7e117e26b221bf6a0d2: Waiting for close lock at 1733937782753Disabling compacts and flushes for region at 1733937782753Disabling writes for close at 1733937782754 (+1 ms)Writing region close event to WAL at 1733937782754Closed at 1733937782754 2024-12-11T17:23:02,757 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:23:02,757 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733937782757"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937782757"}]},"ts":"1733937782757"} 2024-12-11T17:23:02,761 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:23:02,762 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:23:02,762 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937782762"}]},"ts":"1733937782762"} 2024-12-11T17:23:02,765 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-11T17:23:02,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=cb0260ace7c4b7e117e26b221bf6a0d2, ASSIGN}] 2024-12-11T17:23:02,767 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=cb0260ace7c4b7e117e26b221bf6a0d2, ASSIGN 2024-12-11T17:23:02,769 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=cb0260ace7c4b7e117e26b221bf6a0d2, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:23:02,919 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=cb0260ace7c4b7e117e26b221bf6a0d2, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:02,923 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=cb0260ace7c4b7e117e26b221bf6a0d2, ASSIGN because future has completed 2024-12-11T17:23:02,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb0260ace7c4b7e117e26b221bf6a0d2, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:23:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-11T17:23:03,081 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:03,081 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => cb0260ace7c4b7e117e26b221bf6a0d2, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:23:03,082 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,082 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:03,082 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,082 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,084 INFO [StoreOpener-cb0260ace7c4b7e117e26b221bf6a0d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,086 INFO [StoreOpener-cb0260ace7c4b7e117e26b221bf6a0d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb0260ace7c4b7e117e26b221bf6a0d2 columnFamilyName testFamily 2024-12-11T17:23:03,087 DEBUG [StoreOpener-cb0260ace7c4b7e117e26b221bf6a0d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:03,087 INFO [StoreOpener-cb0260ace7c4b7e117e26b221bf6a0d2-1 {}] regionserver.HStore(327): Store=cb0260ace7c4b7e117e26b221bf6a0d2/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:23:03,088 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,089 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,089 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,090 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,090 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,092 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,095 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:23:03,096 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened cb0260ace7c4b7e117e26b221bf6a0d2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66459476, jitterRate=-0.009676635265350342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:23:03,096 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:03,097 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for cb0260ace7c4b7e117e26b221bf6a0d2: Running coprocessor pre-open hook at 1733937783082Writing region info on filesystem at 1733937783082Initializing all the Stores at 1733937783083 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937783083Cleaning up temporary data from old regions at 1733937783090 (+7 ms)Running coprocessor post-open hooks at 1733937783096 (+6 ms)Region opened successfully at 1733937783096 2024-12-11T17:23:03,098 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., pid=18, masterSystemTime=1733937783076 2024-12-11T17:23:03,100 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:03,101 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:03,101 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=cb0260ace7c4b7e117e26b221bf6a0d2, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:03,104 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb0260ace7c4b7e117e26b221bf6a0d2, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:23:03,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-11T17:23:03,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure cb0260ace7c4b7e117e26b221bf6a0d2, server=790ba370077b,39119,1733937725853 in 182 msec 2024-12-11T17:23:03,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-11T17:23:03,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=cb0260ace7c4b7e117e26b221bf6a0d2, ASSIGN in 343 msec 2024-12-11T17:23:03,113 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:23:03,114 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937783113"}]},"ts":"1733937783113"} 2024-12-11T17:23:03,116 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-11T17:23:03,118 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:23:03,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.1970 sec 2024-12-11T17:23:04,033 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T17:23:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-11T17:23:04,973 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-11T17:23:04,974 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-11T17:23:04,974 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:04,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-11T17:23:04,978 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:04,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-11T17:23:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:04,991 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:23:04,992 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-11T17:23:04,995 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-11T17:23:04,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-11T17:23:05,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-11T17:23:05,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-11T17:23:05,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-11T17:23:05,417 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-11T17:23:05,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=790ba370077b:39119 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-11T17:23:05,999 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1004ms, rpcTimeout=1000ms 2024-12-11T17:23:06,000 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2 from cache 2024-12-11T17:23:06,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-12-11T17:23:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:07,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-11T17:23:07,095 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T17:23:08,000 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=790ba370077b:39119 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=978ms 2024-12-11T17:23:08,000 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=978ms 2024-12-11T17:23:08,000 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., hostname=790ba370077b,39119,1733937725853, seqNum=2 from cache 2024-12-11T17:23:08,008 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 228), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=504 (was 539), ProcessCount=11 (was 11), AvailableMemoryMB=4363 (was 4541) 2024-12-11T17:23:08,018 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=504, ProcessCount=11, AvailableMemoryMB=4363 2024-12-11T17:23:08,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:23:08,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-11T17:23:08,024 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:23:08,024 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:08,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-11T17:23:08,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T17:23:08,026 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:23:08,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741845_1021 (size=416) 2024-12-11T17:23:08,437 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e043a4cd8e7679604b653d3163c5bf08, NAME => 'testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:23:08,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741846_1022 (size=67) 2024-12-11T17:23:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-12-11T17:23:08,592 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-11T17:23:08,846 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:08,847 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing e043a4cd8e7679604b653d3163c5bf08, disabling compactions & flushes 2024-12-11T17:23:08,847 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:08,847 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:08,847 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. after waiting 0 ms 2024-12-11T17:23:08,847 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:08,847 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:08,847 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e043a4cd8e7679604b653d3163c5bf08: Waiting for close lock at 1733937788846Disabling compacts and flushes for region at 1733937788846Disabling writes for close at 1733937788847 (+1 ms)Writing region close event to WAL at 1733937788847Closed at 1733937788847 2024-12-11T17:23:08,849 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:23:08,850 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733937788849"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937788849"}]},"ts":"1733937788849"} 2024-12-11T17:23:08,853 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:23:08,855 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:23:08,855 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937788855"}]},"ts":"1733937788855"} 2024-12-11T17:23:08,858 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-11T17:23:08,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e043a4cd8e7679604b653d3163c5bf08, ASSIGN}] 2024-12-11T17:23:08,860 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e043a4cd8e7679604b653d3163c5bf08, ASSIGN 2024-12-11T17:23:08,862 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e043a4cd8e7679604b653d3163c5bf08, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:23:09,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e043a4cd8e7679604b653d3163c5bf08, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:09,017 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e043a4cd8e7679604b653d3163c5bf08, ASSIGN because future has completed 2024-12-11T17:23:09,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e043a4cd8e7679604b653d3163c5bf08, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:23:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T17:23:09,175 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:09,176 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => e043a4cd8e7679604b653d3163c5bf08, NAME => 'testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:23:09,176 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,176 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:09,177 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,177 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,179 INFO [StoreOpener-e043a4cd8e7679604b653d3163c5bf08-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,182 INFO [StoreOpener-e043a4cd8e7679604b653d3163c5bf08-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e043a4cd8e7679604b653d3163c5bf08 columnFamilyName testFamily 2024-12-11T17:23:09,182 DEBUG [StoreOpener-e043a4cd8e7679604b653d3163c5bf08-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:09,183 INFO [StoreOpener-e043a4cd8e7679604b653d3163c5bf08-1 {}] regionserver.HStore(327): Store=e043a4cd8e7679604b653d3163c5bf08/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:23:09,183 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,184 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,185 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,185 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,185 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,188 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,191 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:23:09,192 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened e043a4cd8e7679604b653d3163c5bf08; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63154263, jitterRate=-0.058928146958351135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:23:09,193 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:09,193 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for e043a4cd8e7679604b653d3163c5bf08: Running coprocessor pre-open hook at 1733937789177Writing region info on filesystem at 1733937789177Initializing all the Stores at 1733937789179 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937789179Cleaning up temporary data from old regions at 1733937789186 (+7 ms)Running coprocessor post-open hooks at 1733937789193 (+7 ms)Region opened successfully at 1733937789193 2024-12-11T17:23:09,195 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08., pid=21, masterSystemTime=1733937789170 2024-12-11T17:23:09,198 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:09,199 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:09,199 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=e043a4cd8e7679604b653d3163c5bf08, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:09,202 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure e043a4cd8e7679604b653d3163c5bf08, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:23:09,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-11T17:23:09,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure e043a4cd8e7679604b653d3163c5bf08, server=790ba370077b,39119,1733937725853 in 187 msec 2024-12-11T17:23:09,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-11T17:23:09,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=e043a4cd8e7679604b653d3163c5bf08, ASSIGN in 350 msec 2024-12-11T17:23:09,213 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:23:09,214 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937789213"}]},"ts":"1733937789213"} 2024-12-11T17:23:09,216 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-11T17:23:09,218 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:23:09,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.1970 sec 2024-12-11T17:23:11,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T17:23:11,044 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-11T17:23:11,044 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-11T17:23:11,045 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:11,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-11T17:23:11,051 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:11,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-11T17:23:11,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:11,072 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:23:11,072 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-11T17:23:11,078 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-11T17:23:11,081 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-11T17:23:11,083 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-11T17:23:11,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-11T17:23:14,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39119 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975064' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-11T17:23:14,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39119 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:51980 deadline: 1733937795090, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975064' does not exist 2024-12-11T17:23:14,094 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 790ba370077b,39119,1733937725853 for closing scanner id = -6870863297436975064 for e043a4cd8e7679604b653d3163c5bf08 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975064' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T17:23:14,101 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=479 (was 504), ProcessCount=11 (was 11), AvailableMemoryMB=4242 (was 4363) 2024-12-11T17:23:14,110 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=479, ProcessCount=11, AvailableMemoryMB=4240 2024-12-11T17:23:14,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:23:14,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-11T17:23:14,116 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:23:14,116 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:14,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-11T17:23:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-11T17:23:14,117 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:23:14,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741847_1023 (size=411) 2024-12-11T17:23:14,127 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7020c7471ead5a54b29fdf76fd801b1a, NAME => 'testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:23:14,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741848_1024 (size=62) 2024-12-11T17:23:14,537 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:14,537 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 7020c7471ead5a54b29fdf76fd801b1a, disabling compactions & flushes 2024-12-11T17:23:14,537 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:14,537 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:14,537 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. after waiting 0 ms 2024-12-11T17:23:14,537 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:14,537 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:14,537 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7020c7471ead5a54b29fdf76fd801b1a: Waiting for close lock at 1733937794537Disabling compacts and flushes for region at 1733937794537Disabling writes for close at 1733937794537Writing region close event to WAL at 1733937794537Closed at 1733937794537 2024-12-11T17:23:14,539 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:23:14,539 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733937794539"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937794539"}]},"ts":"1733937794539"} 2024-12-11T17:23:14,543 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:23:14,545 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:23:14,545 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937794545"}]},"ts":"1733937794545"} 2024-12-11T17:23:14,549 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-11T17:23:14,550 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7020c7471ead5a54b29fdf76fd801b1a, ASSIGN}] 2024-12-11T17:23:14,552 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7020c7471ead5a54b29fdf76fd801b1a, ASSIGN 2024-12-11T17:23:14,554 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7020c7471ead5a54b29fdf76fd801b1a, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:23:14,586 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-11T17:23:14,678 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T17:23:14,705 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=7020c7471ead5a54b29fdf76fd801b1a, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:14,709 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7020c7471ead5a54b29fdf76fd801b1a, ASSIGN because future has completed 2024-12-11T17:23:14,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7020c7471ead5a54b29fdf76fd801b1a, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:23:14,712 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-11T17:23:14,875 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:14,876 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 7020c7471ead5a54b29fdf76fd801b1a, NAME => 'testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:23:14,876 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,877 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:14,877 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,877 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,880 INFO [StoreOpener-7020c7471ead5a54b29fdf76fd801b1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,884 INFO [StoreOpener-7020c7471ead5a54b29fdf76fd801b1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7020c7471ead5a54b29fdf76fd801b1a columnFamilyName testFamily 2024-12-11T17:23:14,884 DEBUG [StoreOpener-7020c7471ead5a54b29fdf76fd801b1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:14,885 INFO [StoreOpener-7020c7471ead5a54b29fdf76fd801b1a-1 {}] regionserver.HStore(327): Store=7020c7471ead5a54b29fdf76fd801b1a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:23:14,886 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,888 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,890 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,891 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,891 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,894 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,897 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:23:14,898 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 7020c7471ead5a54b29fdf76fd801b1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62011841, jitterRate=-0.07595156133174896}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:23:14,898 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:14,899 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 7020c7471ead5a54b29fdf76fd801b1a: Running coprocessor pre-open hook at 1733937794877Writing region info on filesystem at 1733937794877Initializing all the Stores at 1733937794879 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937794880 (+1 ms)Cleaning up temporary data from old regions at 1733937794891 (+11 ms)Running coprocessor post-open hooks at 1733937794898 (+7 ms)Region opened successfully at 1733937794899 (+1 ms) 2024-12-11T17:23:14,900 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a., pid=24, masterSystemTime=1733937794864 2024-12-11T17:23:14,903 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:14,903 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:14,904 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=7020c7471ead5a54b29fdf76fd801b1a, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:14,907 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7020c7471ead5a54b29fdf76fd801b1a, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:23:14,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-11T17:23:14,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 7020c7471ead5a54b29fdf76fd801b1a, server=790ba370077b,39119,1733937725853 in 198 msec 2024-12-11T17:23:14,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-11T17:23:14,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=7020c7471ead5a54b29fdf76fd801b1a, ASSIGN in 361 msec 2024-12-11T17:23:14,916 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:23:14,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937794916"}]},"ts":"1733937794916"} 2024-12-11T17:23:14,919 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-11T17:23:14,921 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:23:14,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 809 msec 2024-12-11T17:23:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-11T17:23:15,134 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-11T17:23:15,134 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-11T17:23:15,135 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:15,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-11T17:23:15,140 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:15,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-11T17:23:15,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:15,155 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:23:15,155 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-11T17:23:15,158 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-11T17:23:15,160 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-11T17:23:15,161 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-11T17:23:15,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-11T17:23:15,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-11T17:23:15,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-11T17:23:15,418 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-11T17:23:15,418 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T17:23:18,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39119 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975060' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-11T17:23:18,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39119 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:51980 deadline: 1733937799170, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975060' does not exist 2024-12-11T17:23:18,175 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 790ba370077b,39119,1733937725853 for closing scanner id = -6870863297436975060 for 7020c7471ead5a54b29fdf76fd801b1a of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975060' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T17:23:18,182 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=473 (was 479), ProcessCount=11 (was 11), AvailableMemoryMB=4126 (was 4240) 2024-12-11T17:23:18,192 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=473, ProcessCount=11, AvailableMemoryMB=4126 2024-12-11T17:23:18,196 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:23:18,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-11T17:23:18,201 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:23:18,202 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:18,202 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-11T17:23:18,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T17:23:18,205 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:23:18,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741849_1025 (size=419) 2024-12-11T17:23:18,631 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b0d32ac8e8a59c7156e4cf9ea1cb7a23, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:23:18,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741850_1026 (size=70) 2024-12-11T17:23:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-12-11T17:23:19,046 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:19,046 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing b0d32ac8e8a59c7156e4cf9ea1cb7a23, disabling compactions & flushes 2024-12-11T17:23:19,046 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:19,047 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:19,047 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. after waiting 0 ms 2024-12-11T17:23:19,047 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:19,047 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:19,047 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for b0d32ac8e8a59c7156e4cf9ea1cb7a23: Waiting for close lock at 1733937799046Disabling compacts and flushes for region at 1733937799046Disabling writes for close at 1733937799047 (+1 ms)Writing region close event to WAL at 1733937799047Closed at 1733937799047 2024-12-11T17:23:19,050 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:23:19,050 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733937799050"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937799050"}]},"ts":"1733937799050"} 2024-12-11T17:23:19,054 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:23:19,055 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:23:19,056 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937799055"}]},"ts":"1733937799055"} 2024-12-11T17:23:19,058 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-11T17:23:19,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b0d32ac8e8a59c7156e4cf9ea1cb7a23, ASSIGN}] 2024-12-11T17:23:19,060 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b0d32ac8e8a59c7156e4cf9ea1cb7a23, ASSIGN 2024-12-11T17:23:19,062 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b0d32ac8e8a59c7156e4cf9ea1cb7a23, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:23:19,213 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b0d32ac8e8a59c7156e4cf9ea1cb7a23, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T17:23:19,216 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b0d32ac8e8a59c7156e4cf9ea1cb7a23, ASSIGN because future has completed 2024-12-11T17:23:19,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0d32ac8e8a59c7156e4cf9ea1cb7a23, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:23:19,312 INFO [master/790ba370077b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-11T17:23:19,312 INFO [master/790ba370077b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-11T17:23:19,374 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:19,375 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => b0d32ac8e8a59c7156e4cf9ea1cb7a23, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:23:19,375 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,375 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:19,375 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,375 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,378 INFO [StoreOpener-b0d32ac8e8a59c7156e4cf9ea1cb7a23-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,380 INFO [StoreOpener-b0d32ac8e8a59c7156e4cf9ea1cb7a23-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b0d32ac8e8a59c7156e4cf9ea1cb7a23 columnFamilyName testFamily 2024-12-11T17:23:19,381 DEBUG [StoreOpener-b0d32ac8e8a59c7156e4cf9ea1cb7a23-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:19,382 INFO [StoreOpener-b0d32ac8e8a59c7156e4cf9ea1cb7a23-1 {}] regionserver.HStore(327): Store=b0d32ac8e8a59c7156e4cf9ea1cb7a23/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:23:19,382 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,384 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,385 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,386 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,387 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,390 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,397 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:23:19,398 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened b0d32ac8e8a59c7156e4cf9ea1cb7a23; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65308437, jitterRate=-0.026828452944755554}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:23:19,398 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:19,399 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for b0d32ac8e8a59c7156e4cf9ea1cb7a23: Running coprocessor pre-open hook at 1733937799376Writing region info on filesystem at 1733937799376Initializing all the Stores at 1733937799377 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937799377Cleaning up temporary data from old regions at 1733937799387 (+10 ms)Running coprocessor post-open hooks at 1733937799398 (+11 ms)Region opened successfully at 1733937799399 (+1 ms) 2024-12-11T17:23:19,401 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., pid=27, masterSystemTime=1733937799369 2024-12-11T17:23:19,405 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:19,405 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:19,407 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=b0d32ac8e8a59c7156e4cf9ea1cb7a23, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:19,415 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure b0d32ac8e8a59c7156e4cf9ea1cb7a23, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:23:19,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-11T17:23:19,420 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure b0d32ac8e8a59c7156e4cf9ea1cb7a23, server=790ba370077b,39119,1733937725853 in 201 msec 2024-12-11T17:23:19,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-11T17:23:19,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=b0d32ac8e8a59c7156e4cf9ea1cb7a23, ASSIGN in 361 msec 2024-12-11T17:23:19,425 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:23:19,426 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937799425"}]},"ts":"1733937799425"} 2024-12-11T17:23:19,432 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-11T17:23:19,434 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:23:19,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2380 sec 2024-12-11T17:23:20,379 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T17:23:20,713 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-11T17:23:20,714 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-11T17:23:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T17:23:21,234 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-11T17:23:21,234 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-11T17:23:21,234 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:21,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-11T17:23:21,239 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:21,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-11T17:23:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:21,254 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:23:21,254 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-11T17:23:21,258 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-11T17:23:21,259 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-11T17:23:25,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-11T17:23:25,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-11T17:23:27,260 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., hostname=790ba370077b,39119,1733937725853, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., hostname=790ba370077b,39119,1733937725853, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=790ba370077b:39119 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-12-11T17:23:27,260 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., hostname=790ba370077b,39119,1733937725853, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-12-11T17:23:27,261 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., hostname=790ba370077b,39119,1733937725853, seqNum=2 from cache 2024-12-11T17:23:27,277 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=415 (was 473), ProcessCount=11 (was 11), AvailableMemoryMB=4093 (was 4126) 2024-12-11T17:23:27,284 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=415, ProcessCount=11, AvailableMemoryMB=4093 2024-12-11T17:23:27,287 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:23:27,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-11T17:23:27,291 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:23:27,291 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:27,291 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-11T17:23:27,292 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:23:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T17:23:27,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741851_1027 (size=407) 2024-12-11T17:23:27,704 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e4e2079ce47c7fdcecb48a79a69d3403, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:23:27,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741852_1028 (size=58) 2024-12-11T17:23:27,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-12-11T17:23:28,114 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:28,114 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing e4e2079ce47c7fdcecb48a79a69d3403, disabling compactions & flushes 2024-12-11T17:23:28,114 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:28,115 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:28,115 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. after waiting 0 ms 2024-12-11T17:23:28,115 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:28,115 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:28,115 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for e4e2079ce47c7fdcecb48a79a69d3403: Waiting for close lock at 1733937808114Disabling compacts and flushes for region at 1733937808114Disabling writes for close at 1733937808115 (+1 ms)Writing region close event to WAL at 1733937808115Closed at 1733937808115 2024-12-11T17:23:28,118 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:23:28,118 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733937808118"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937808118"}]},"ts":"1733937808118"} 2024-12-11T17:23:28,122 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:23:28,123 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:23:28,123 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937808123"}]},"ts":"1733937808123"} 2024-12-11T17:23:28,126 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-11T17:23:28,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e4e2079ce47c7fdcecb48a79a69d3403, ASSIGN}] 2024-12-11T17:23:28,128 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e4e2079ce47c7fdcecb48a79a69d3403, ASSIGN 2024-12-11T17:23:28,129 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e4e2079ce47c7fdcecb48a79a69d3403, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:23:28,280 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e4e2079ce47c7fdcecb48a79a69d3403, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:28,284 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e4e2079ce47c7fdcecb48a79a69d3403, ASSIGN because future has completed 2024-12-11T17:23:28,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4e2079ce47c7fdcecb48a79a69d3403, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:23:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T17:23:28,446 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:28,446 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => e4e2079ce47c7fdcecb48a79a69d3403, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:23:28,447 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,447 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:28,448 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,448 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,451 INFO [StoreOpener-e4e2079ce47c7fdcecb48a79a69d3403-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,454 INFO [StoreOpener-e4e2079ce47c7fdcecb48a79a69d3403-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4e2079ce47c7fdcecb48a79a69d3403 columnFamilyName testFamily 2024-12-11T17:23:28,454 DEBUG [StoreOpener-e4e2079ce47c7fdcecb48a79a69d3403-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:28,455 INFO [StoreOpener-e4e2079ce47c7fdcecb48a79a69d3403-1 {}] regionserver.HStore(327): Store=e4e2079ce47c7fdcecb48a79a69d3403/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:23:28,455 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,457 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,458 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,459 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,459 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,463 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,468 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:23:28,469 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened e4e2079ce47c7fdcecb48a79a69d3403; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72887651, jitterRate=0.08611063659191132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:23:28,470 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:28,470 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for e4e2079ce47c7fdcecb48a79a69d3403: Running coprocessor pre-open hook at 1733937808448Writing region info on filesystem at 1733937808448Initializing all the Stores at 1733937808450 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937808450Cleaning up temporary data from old regions at 1733937808459 (+9 ms)Running coprocessor post-open hooks at 1733937808470 (+11 ms)Region opened successfully at 1733937808470 2024-12-11T17:23:28,472 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403., pid=30, masterSystemTime=1733937808438 2024-12-11T17:23:28,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:28,476 INFO [RS_OPEN_PRIORITY_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:28,477 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=e4e2079ce47c7fdcecb48a79a69d3403, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:28,480 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4e2079ce47c7fdcecb48a79a69d3403, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:23:28,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-11T17:23:28,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure e4e2079ce47c7fdcecb48a79a69d3403, server=790ba370077b,39119,1733937725853 in 197 msec 2024-12-11T17:23:28,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-11T17:23:28,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=e4e2079ce47c7fdcecb48a79a69d3403, ASSIGN in 360 msec 2024-12-11T17:23:28,491 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:23:28,491 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937808491"}]},"ts":"1733937808491"} 2024-12-11T17:23:28,495 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-11T17:23:28,496 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:23:28,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2090 sec 2024-12-11T17:23:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-11T17:23:30,314 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-11T17:23:30,315 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-11T17:23:30,315 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:30,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-11T17:23:30,324 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:30,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-11T17:23:30,327 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:30,338 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:23:30,338 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-11T17:23:30,342 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-11T17:23:30,348 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-11T17:23:30,350 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-11T17:23:30,351 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-11T17:23:32,717 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-11T17:23:33,978 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T17:23:34,033 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T17:23:35,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-11T17:23:35,416 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-11T17:23:36,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39119 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975052' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-11T17:23:36,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39119 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.3:51980 deadline: 1733937822360, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975052' does not exist 2024-12-11T17:23:36,363 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 790ba370077b,39119,1733937725853 for closing scanner id = -6870863297436975052 for e4e2079ce47c7fdcecb48a79a69d3403 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6870863297436975052' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T17:23:36,367 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=462 (was 415) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4783 (was 4093) - AvailableMemoryMB LEAK? - 2024-12-11T17:23:36,374 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=462, ProcessCount=11, AvailableMemoryMB=4783 2024-12-11T17:23:36,376 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T17:23:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-11T17:23:36,380 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T17:23:36,380 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:36,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-11T17:23:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T17:23:36,381 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T17:23:36,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741853_1029 (size=423) 2024-12-11T17:23:36,793 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 43a788ca58b94f9514910b987039b419, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90 2024-12-11T17:23:36,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741854_1030 (size=74) 2024-12-11T17:23:36,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-12-11T17:23:37,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:37,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 43a788ca58b94f9514910b987039b419, disabling compactions & flushes 2024-12-11T17:23:37,213 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:37,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:37,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. after waiting 0 ms 2024-12-11T17:23:37,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:37,213 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:37,213 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 43a788ca58b94f9514910b987039b419: Waiting for close lock at 1733937817213Disabling compacts and flushes for region at 1733937817213Disabling writes for close at 1733937817213Writing region close event to WAL at 1733937817213Closed at 1733937817213 2024-12-11T17:23:37,215 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T17:23:37,215 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733937817215"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733937817215"}]},"ts":"1733937817215"} 2024-12-11T17:23:37,218 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T17:23:37,219 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T17:23:37,219 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937817219"}]},"ts":"1733937817219"} 2024-12-11T17:23:37,222 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-11T17:23:37,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=43a788ca58b94f9514910b987039b419, ASSIGN}] 2024-12-11T17:23:37,224 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=43a788ca58b94f9514910b987039b419, ASSIGN 2024-12-11T17:23:37,225 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=43a788ca58b94f9514910b987039b419, ASSIGN; state=OFFLINE, location=790ba370077b,39119,1733937725853; forceNewPlan=false, retain=false 2024-12-11T17:23:37,376 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=43a788ca58b94f9514910b987039b419, regionState=OPENING, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:37,379 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=43a788ca58b94f9514910b987039b419, ASSIGN because future has completed 2024-12-11T17:23:37,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43a788ca58b94f9514910b987039b419, server=790ba370077b,39119,1733937725853}] 2024-12-11T17:23:37,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T17:23:37,538 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:37,539 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 43a788ca58b94f9514910b987039b419, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419.', STARTKEY => '', ENDKEY => ''} 2024-12-11T17:23:37,539 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,539 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T17:23:37,539 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,539 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,541 INFO [StoreOpener-43a788ca58b94f9514910b987039b419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,543 INFO [StoreOpener-43a788ca58b94f9514910b987039b419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43a788ca58b94f9514910b987039b419 columnFamilyName testFamily 2024-12-11T17:23:37,543 DEBUG [StoreOpener-43a788ca58b94f9514910b987039b419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T17:23:37,544 INFO [StoreOpener-43a788ca58b94f9514910b987039b419-1 {}] regionserver.HStore(327): Store=43a788ca58b94f9514910b987039b419/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T17:23:37,544 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,545 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,545 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,546 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,546 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,548 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,551 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T17:23:37,551 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 43a788ca58b94f9514910b987039b419; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68152598, jitterRate=0.01555284857749939}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-11T17:23:37,551 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:37,552 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 43a788ca58b94f9514910b987039b419: Running coprocessor pre-open hook at 1733937817540Writing region info on filesystem at 1733937817540Initializing all the Stores at 1733937817541 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733937817541Cleaning up temporary data from old regions at 1733937817546 (+5 ms)Running coprocessor post-open hooks at 1733937817551 (+5 ms)Region opened successfully at 1733937817552 (+1 ms) 2024-12-11T17:23:37,553 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., pid=33, masterSystemTime=1733937817533 2024-12-11T17:23:37,556 DEBUG [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:37,556 INFO [RS_OPEN_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:37,556 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=43a788ca58b94f9514910b987039b419, regionState=OPEN, openSeqNum=2, regionLocation=790ba370077b,39119,1733937725853 2024-12-11T17:23:37,559 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43a788ca58b94f9514910b987039b419, server=790ba370077b,39119,1733937725853 because future has completed 2024-12-11T17:23:37,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-11T17:23:37,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 43a788ca58b94f9514910b987039b419, server=790ba370077b,39119,1733937725853 in 181 msec 2024-12-11T17:23:37,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-11T17:23:37,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=43a788ca58b94f9514910b987039b419, ASSIGN in 342 msec 2024-12-11T17:23:37,568 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T17:23:37,568 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733937817568"}]},"ts":"1733937817568"} 2024-12-11T17:23:37,570 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-11T17:23:37,571 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T17:23:37,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.1950 sec 2024-12-11T17:23:38,720 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-11T17:23:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38569 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-11T17:23:39,413 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-11T17:23:39,413 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-11T17:23:39,413 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:39,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-11T17:23:39,418 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-11T17:23:39,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-11T17:23:39,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:39,433 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-11T17:23:39,433 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-11T17:23:39,436 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-11T17:23:39,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-11T17:23:40,006 DEBUG [master/790ba370077b:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-11T17:23:40,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=790ba370077b:39119 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-12-11T17:23:40,440 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-12-11T17:23:40,440 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2 from cache 2024-12-11T17:23:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-12-11T17:23:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2] 2024-12-11T17:23:41,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39119 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-11T17:23:42,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=790ba370077b:39119 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=983ms, rpcTimeout=979ms 2024-12-11T17:23:42,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=983ms, rpcTimeout=979ms 2024-12-11T17:23:42,439 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., hostname=790ba370077b,39119,1733937725853, seqNum=2 from cache 2024-12-11T17:23:42,449 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=441 (was 462), ProcessCount=11 (was 11), AvailableMemoryMB=4657 (was 4783) 2024-12-11T17:23:42,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T17:23:42,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T17:23:42,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:23:42,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:23:42,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T17:23:42,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-11T17:23:42,454 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T17:23:42,455 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T17:23:42,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:23:42,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:23:42,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T17:23:42,455 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-11T17:23:42,455 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2119115424, stopped=false 2024-12-11T17:23:42,455 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=790ba370077b,38569,1733937724982 2024-12-11T17:23:42,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T17:23:42,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T17:23:42,485 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T17:23:42,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:23:42,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:23:42,486 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T17:23:42,486 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T17:23:42,487 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T17:23:42,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:23:42,487 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T17:23:42,487 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '790ba370077b,39119,1733937725853' ***** 2024-12-11T17:23:42,488 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T17:23:42,488 INFO [RS:0;790ba370077b:39119 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-11T17:23:42,488 INFO [RS:0;790ba370077b:39119 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T17:23:42,488 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T17:23:42,488 INFO [RS:0;790ba370077b:39119 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for 6aa8aadf7fa2b36d8b0c503f2cd10a64 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for e043a4cd8e7679604b653d3163c5bf08 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for 7020c7471ead5a54b29fdf76fd801b1a 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for 1be97a5d392c877b9bc9b2732b8cfea7 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for b0d32ac8e8a59c7156e4cf9ea1cb7a23 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for 43a788ca58b94f9514910b987039b419 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for 30d92a4bafdc46c54d1ef5b985214c88 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for 88c575eda49795e049781ca96698701c 2024-12-11T17:23:42,489 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(959): stopping server 790ba370077b,39119,1733937725853 2024-12-11T17:23:42,490 INFO [RS:0;790ba370077b:39119 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T17:23:42,490 INFO [RS:0;790ba370077b:39119 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;790ba370077b:39119. 2024-12-11T17:23:42,490 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6aa8aadf7fa2b36d8b0c503f2cd10a64, disabling compactions & flushes 2024-12-11T17:23:42,490 DEBUG [RS:0;790ba370077b:39119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T17:23:42,490 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:23:42,490 DEBUG [RS:0;790ba370077b:39119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:23:42,490 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:23:42,490 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. after waiting 0 ms 2024-12-11T17:23:42,490 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:23:42,490 INFO [RS:0;790ba370077b:39119 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T17:23:42,490 INFO [RS:0;790ba370077b:39119 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T17:23:42,490 INFO [RS:0;790ba370077b:39119 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T17:23:42,490 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T17:23:42,490 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-11T17:23:42,491 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T17:23:42,491 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T17:23:42,491 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T17:23:42,491 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T17:23:42,491 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T17:23:42,491 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 6aa8aadf7fa2b36d8b0c503f2cd10a64, 7020c7471ead5a54b29fdf76fd801b1a, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2, e043a4cd8e7679604b653d3163c5bf08, e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:42,494 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6aa8aadf7fa2b36d8b0c503f2cd10a64 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:42,494 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-11T17:23:42,563 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64/.tmp/testFamily/2007f4cb08284679b563fc23c0bf17c0 is 61, key is row-0/testFamily:testQualifier/1733937762526/Put/seqid=0 2024-12-11T17:23:42,567 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/info/9a50ad2bdc8d40a5bc96fb0cf9448a18 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64./info:regioninfo/1733937760674/Put/seqid=0 2024-12-11T17:23:42,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741855_1031 (size=20564) 2024-12-11T17:23:42,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741856_1032 (size=5049) 2024-12-11T17:23:42,691 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 6aa8aadf7fa2b36d8b0c503f2cd10a64, 7020c7471ead5a54b29fdf76fd801b1a, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2, e043a4cd8e7679604b653d3163c5bf08, e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:42,892 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 6aa8aadf7fa2b36d8b0c503f2cd10a64, 7020c7471ead5a54b29fdf76fd801b1a, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2, e043a4cd8e7679604b653d3163c5bf08, e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:42,958 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-11T17:23:42,981 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/info/9a50ad2bdc8d40a5bc96fb0cf9448a18 2024-12-11T17:23:42,987 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64/.tmp/testFamily/2007f4cb08284679b563fc23c0bf17c0 2024-12-11T17:23:43,041 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T17:23:43,058 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64/.tmp/testFamily/2007f4cb08284679b563fc23c0bf17c0 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64/testFamily/2007f4cb08284679b563fc23c0bf17c0 2024-12-11T17:23:43,076 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/ns/de731320b1344c729c824d9d639f570c is 43, key is default/ns:d/1733937759186/Put/seqid=0 2024-12-11T17:23:43,078 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64/testFamily/2007f4cb08284679b563fc23c0bf17c0, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:43,092 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 6aa8aadf7fa2b36d8b0c503f2cd10a64, 7020c7471ead5a54b29fdf76fd801b1a, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2, e043a4cd8e7679604b653d3163c5bf08, e4e2079ce47c7fdcecb48a79a69d3403 2024-12-11T17:23:43,095 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6aa8aadf7fa2b36d8b0c503f2cd10a64 in 598ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:43,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741857_1033 (size=5153) 2024-12-11T17:23:43,108 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextExceptionAsync/6aa8aadf7fa2b36d8b0c503f2cd10a64/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:43,111 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:23:43,111 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6aa8aadf7fa2b36d8b0c503f2cd10a64: Waiting for close lock at 1733937822489Running coprocessor pre-close hooks at 1733937822490 (+1 ms)Disabling compacts and flushes for region at 1733937822490Disabling writes for close at 1733937822490Obtaining lock to block concurrent updates at 1733937822494 (+4 ms)Preparing flush snapshotting stores in 6aa8aadf7fa2b36d8b0c503f2cd10a64 at 1733937822494Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937822502 (+8 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. at 1733937822504 (+2 ms)Flushing 6aa8aadf7fa2b36d8b0c503f2cd10a64/testFamily: creating writer at 1733937822506 (+2 ms)Flushing 6aa8aadf7fa2b36d8b0c503f2cd10a64/testFamily: appending metadata at 1733937822555 (+49 ms)Flushing 6aa8aadf7fa2b36d8b0c503f2cd10a64/testFamily: closing flushed file at 1733937822559 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69eff78c: reopening flushed file at 1733937823046 (+487 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6aa8aadf7fa2b36d8b0c503f2cd10a64 in 598ms, sequenceid=8, compaction requested=false at 1733937823095 (+49 ms)Writing region close event to WAL at 1733937823102 (+7 ms)Running coprocessor post-close hooks at 1733937823110 (+8 ms)Closed at 1733937823111 (+1 ms) 2024-12-11T17:23:43,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733937759432.6aa8aadf7fa2b36d8b0c503f2cd10a64. 2024-12-11T17:23:43,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e4e2079ce47c7fdcecb48a79a69d3403, disabling compactions & flushes 2024-12-11T17:23:43,112 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:43,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:43,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. after waiting 0 ms 2024-12-11T17:23:43,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:43,112 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e4e2079ce47c7fdcecb48a79a69d3403 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:43,137 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403/.tmp/testFamily/2cf8c7a9a18f4a489b7bff095ea99b33 is 61, key is row-0/testFamily:testQualifier/1733937810328/Put/seqid=0 2024-12-11T17:23:43,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741858_1034 (size=5049) 2024-12-11T17:23:43,151 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403/.tmp/testFamily/2cf8c7a9a18f4a489b7bff095ea99b33 2024-12-11T17:23:43,166 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403/.tmp/testFamily/2cf8c7a9a18f4a489b7bff095ea99b33 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403/testFamily/2cf8c7a9a18f4a489b7bff095ea99b33 2024-12-11T17:23:43,184 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403/testFamily/2cf8c7a9a18f4a489b7bff095ea99b33, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:43,187 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e4e2079ce47c7fdcecb48a79a69d3403 in 75ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:43,198 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNext/e4e2079ce47c7fdcecb48a79a69d3403/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:43,199 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:43,199 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e4e2079ce47c7fdcecb48a79a69d3403: Waiting for close lock at 1733937823112Running coprocessor pre-close hooks at 1733937823112Disabling compacts and flushes for region at 1733937823112Disabling writes for close at 1733937823112Obtaining lock to block concurrent updates at 1733937823112Preparing flush snapshotting stores in e4e2079ce47c7fdcecb48a79a69d3403 at 1733937823112Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937823113 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. at 1733937823113Flushing e4e2079ce47c7fdcecb48a79a69d3403/testFamily: creating writer at 1733937823113Flushing e4e2079ce47c7fdcecb48a79a69d3403/testFamily: appending metadata at 1733937823137 (+24 ms)Flushing e4e2079ce47c7fdcecb48a79a69d3403/testFamily: closing flushed file at 1733937823137Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c31e6e2: reopening flushed file at 1733937823161 (+24 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e4e2079ce47c7fdcecb48a79a69d3403 in 75ms, sequenceid=8, compaction requested=false at 1733937823187 (+26 ms)Writing region close event to WAL at 1733937823188 (+1 ms)Running coprocessor post-close hooks at 1733937823198 (+10 ms)Closed at 1733937823198 2024-12-11T17:23:43,204 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733937807287.e4e2079ce47c7fdcecb48a79a69d3403. 2024-12-11T17:23:43,205 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e043a4cd8e7679604b653d3163c5bf08, disabling compactions & flushes 2024-12-11T17:23:43,205 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:43,205 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:43,205 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. after waiting 0 ms 2024-12-11T17:23:43,205 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:43,205 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e043a4cd8e7679604b653d3163c5bf08 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:43,235 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08/.tmp/testFamily/12e8a5975fde4eaab986783c212d9ff8 is 61, key is row-0/testFamily:testQualifier/1733937791059/Put/seqid=0 2024-12-11T17:23:43,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741859_1035 (size=5049) 2024-12-11T17:23:43,241 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08/.tmp/testFamily/12e8a5975fde4eaab986783c212d9ff8 2024-12-11T17:23:43,253 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08/.tmp/testFamily/12e8a5975fde4eaab986783c212d9ff8 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08/testFamily/12e8a5975fde4eaab986783c212d9ff8 2024-12-11T17:23:43,263 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08/testFamily/12e8a5975fde4eaab986783c212d9ff8, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:43,264 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e043a4cd8e7679604b653d3163c5bf08 in 59ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:43,269 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNextAsync/e043a4cd8e7679604b653d3163c5bf08/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:43,270 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:43,271 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e043a4cd8e7679604b653d3163c5bf08: Waiting for close lock at 1733937823205Running coprocessor pre-close hooks at 1733937823205Disabling compacts and flushes for region at 1733937823205Disabling writes for close at 1733937823205Obtaining lock to block concurrent updates at 1733937823205Preparing flush snapshotting stores in e043a4cd8e7679604b653d3163c5bf08 at 1733937823205Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937823205Flushing stores of testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. at 1733937823207 (+2 ms)Flushing e043a4cd8e7679604b653d3163c5bf08/testFamily: creating writer at 1733937823208 (+1 ms)Flushing e043a4cd8e7679604b653d3163c5bf08/testFamily: appending metadata at 1733937823235 (+27 ms)Flushing e043a4cd8e7679604b653d3163c5bf08/testFamily: closing flushed file at 1733937823235Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@261b068b: reopening flushed file at 1733937823252 (+17 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e043a4cd8e7679604b653d3163c5bf08 in 59ms, sequenceid=8, compaction requested=false at 1733937823264 (+12 ms)Writing region close event to WAL at 1733937823265 (+1 ms)Running coprocessor post-close hooks at 1733937823270 (+5 ms)Closed at 1733937823270 2024-12-11T17:23:43,271 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733937788020.e043a4cd8e7679604b653d3163c5bf08. 2024-12-11T17:23:43,271 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7020c7471ead5a54b29fdf76fd801b1a, disabling compactions & flushes 2024-12-11T17:23:43,271 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:43,271 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:43,271 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. after waiting 0 ms 2024-12-11T17:23:43,271 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:43,271 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7020c7471ead5a54b29fdf76fd801b1a 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:43,290 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a/.tmp/testFamily/72e699e675734dacb00b1cc0932792b1 is 61, key is row-0/testFamily:testQualifier/1733937795144/Put/seqid=0 2024-12-11T17:23:43,293 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 7020c7471ead5a54b29fdf76fd801b1a, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:43,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741860_1036 (size=5049) 2024-12-11T17:23:43,383 INFO [regionserver/790ba370077b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-11T17:23:43,384 INFO [regionserver/790ba370077b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-11T17:23:43,493 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1321): Waiting on 8 regions to close 2024-12-11T17:23:43,493 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1325): Online Regions={7020c7471ead5a54b29fdf76fd801b1a=testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a., 1be97a5d392c877b9bc9b2732b8cfea7=hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7., 1588230740=hbase:meta,,1.1588230740, b0d32ac8e8a59c7156e4cf9ea1cb7a23=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., 43a788ca58b94f9514910b987039b419=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., 30d92a4bafdc46c54d1ef5b985214c88=hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88., cb0260ace7c4b7e117e26b221bf6a0d2=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., 88c575eda49795e049781ca96698701c=testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c.} 2024-12-11T17:23:43,493 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 7020c7471ead5a54b29fdf76fd801b1a, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:43,501 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/ns/de731320b1344c729c824d9d639f570c 2024-12-11T17:23:43,529 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/table/e65a62b6721f43d0ae1bcb6871c99ace is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733937760691/Put/seqid=0 2024-12-11T17:23:43,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741861_1037 (size=6699) 2024-12-11T17:23:43,539 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/table/e65a62b6721f43d0ae1bcb6871c99ace 2024-12-11T17:23:43,551 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/info/9a50ad2bdc8d40a5bc96fb0cf9448a18 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/info/9a50ad2bdc8d40a5bc96fb0cf9448a18 2024-12-11T17:23:43,561 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/info/9a50ad2bdc8d40a5bc96fb0cf9448a18, entries=100, sequenceid=56, filesize=20.1 K 2024-12-11T17:23:43,563 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/ns/de731320b1344c729c824d9d639f570c as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/ns/de731320b1344c729c824d9d639f570c 2024-12-11T17:23:43,573 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/ns/de731320b1344c729c824d9d639f570c, entries=2, sequenceid=56, filesize=5.0 K 2024-12-11T17:23:43,575 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/.tmp/table/e65a62b6721f43d0ae1bcb6871c99ace as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/table/e65a62b6721f43d0ae1bcb6871c99ace 2024-12-11T17:23:43,583 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/table/e65a62b6721f43d0ae1bcb6871c99ace, entries=20, sequenceid=56, filesize=6.5 K 2024-12-11T17:23:43,585 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1094ms, sequenceid=56, compaction requested=false 2024-12-11T17:23:43,591 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-11T17:23:43,593 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T17:23:43,593 INFO [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T17:23:43,593 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733937822490Running coprocessor pre-close hooks at 1733937822491 (+1 ms)Disabling compacts and flushes for region at 1733937822491Disabling writes for close at 1733937822491Obtaining lock to block concurrent updates at 1733937822494 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1733937822494Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733937822502 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733937822504 (+2 ms)Flushing 1588230740/info: creating writer at 1733937822506 (+2 ms)Flushing 1588230740/info: appending metadata at 1733937822563 (+57 ms)Flushing 1588230740/info: closing flushed file at 1733937822563Flushing 1588230740/ns: creating writer at 1733937823046 (+483 ms)Flushing 1588230740/ns: appending metadata at 1733937823074 (+28 ms)Flushing 1588230740/ns: closing flushed file at 1733937823075 (+1 ms)Flushing 1588230740/table: creating writer at 1733937823511 (+436 ms)Flushing 1588230740/table: appending metadata at 1733937823528 (+17 ms)Flushing 1588230740/table: closing flushed file at 1733937823528Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@772c85cb: reopening flushed file at 1733937823549 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c7473dc: reopening flushed file at 1733937823562 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4916b4cd: reopening flushed file at 1733937823573 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1094ms, sequenceid=56, compaction requested=false at 1733937823585 (+12 ms)Writing region close event to WAL at 1733937823587 (+2 ms)Running coprocessor post-close hooks at 1733937823592 (+5 ms)Closed at 1733937823593 (+1 ms) 2024-12-11T17:23:43,593 DEBUG [RS_CLOSE_META-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T17:23:43,694 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 7020c7471ead5a54b29fdf76fd801b1a, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:43,698 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a/.tmp/testFamily/72e699e675734dacb00b1cc0932792b1 2024-12-11T17:23:43,710 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a/.tmp/testFamily/72e699e675734dacb00b1cc0932792b1 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a/testFamily/72e699e675734dacb00b1cc0932792b1 2024-12-11T17:23:43,720 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a/testFamily/72e699e675734dacb00b1cc0932792b1, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:43,721 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7020c7471ead5a54b29fdf76fd801b1a in 450ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:43,732 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnNext/7020c7471ead5a54b29fdf76fd801b1a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:43,734 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:43,734 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7020c7471ead5a54b29fdf76fd801b1a: Waiting for close lock at 1733937823271Running coprocessor pre-close hooks at 1733937823271Disabling compacts and flushes for region at 1733937823271Disabling writes for close at 1733937823271Obtaining lock to block concurrent updates at 1733937823271Preparing flush snapshotting stores in 7020c7471ead5a54b29fdf76fd801b1a at 1733937823271Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937823271Flushing stores of testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. at 1733937823272 (+1 ms)Flushing 7020c7471ead5a54b29fdf76fd801b1a/testFamily: creating writer at 1733937823272Flushing 7020c7471ead5a54b29fdf76fd801b1a/testFamily: appending metadata at 1733937823290 (+18 ms)Flushing 7020c7471ead5a54b29fdf76fd801b1a/testFamily: closing flushed file at 1733937823290Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aeb971e: reopening flushed file at 1733937823708 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7020c7471ead5a54b29fdf76fd801b1a in 450ms, sequenceid=8, compaction requested=false at 1733937823721 (+13 ms)Writing region close event to WAL at 1733937823725 (+4 ms)Running coprocessor post-close hooks at 1733937823734 (+9 ms)Closed at 1733937823734 2024-12-11T17:23:43,734 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733937794112.7020c7471ead5a54b29fdf76fd801b1a. 2024-12-11T17:23:43,735 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1be97a5d392c877b9bc9b2732b8cfea7, disabling compactions & flushes 2024-12-11T17:23:43,735 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:23:43,735 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:23:43,735 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. after waiting 0 ms 2024-12-11T17:23:43,735 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:23:43,735 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1be97a5d392c877b9bc9b2732b8cfea7 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:43,764 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7/.tmp/testFamily/fd007416ddcd4f6dae2072089c7d8a40 is 61, key is row-0/testFamily:testQualifier/1733937765702/Put/seqid=0 2024-12-11T17:23:43,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741862_1038 (size=5049) 2024-12-11T17:23:43,894 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:44,094 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 1be97a5d392c877b9bc9b2732b8cfea7, 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:44,172 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7/.tmp/testFamily/fd007416ddcd4f6dae2072089c7d8a40 2024-12-11T17:23:44,182 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7/.tmp/testFamily/fd007416ddcd4f6dae2072089c7d8a40 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7/testFamily/fd007416ddcd4f6dae2072089c7d8a40 2024-12-11T17:23:44,192 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7/testFamily/fd007416ddcd4f6dae2072089c7d8a40, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:44,194 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1be97a5d392c877b9bc9b2732b8cfea7 in 458ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:44,200 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScanner/1be97a5d392c877b9bc9b2732b8cfea7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:44,200 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:23:44,201 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1be97a5d392c877b9bc9b2732b8cfea7: Waiting for close lock at 1733937823735Running coprocessor pre-close hooks at 1733937823735Disabling compacts and flushes for region at 1733937823735Disabling writes for close at 1733937823735Obtaining lock to block concurrent updates at 1733937823735Preparing flush snapshotting stores in 1be97a5d392c877b9bc9b2732b8cfea7 at 1733937823735Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937823735Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. at 1733937823736 (+1 ms)Flushing 1be97a5d392c877b9bc9b2732b8cfea7/testFamily: creating writer at 1733937823737 (+1 ms)Flushing 1be97a5d392c877b9bc9b2732b8cfea7/testFamily: appending metadata at 1733937823764 (+27 ms)Flushing 1be97a5d392c877b9bc9b2732b8cfea7/testFamily: closing flushed file at 1733937823764Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@525e9379: reopening flushed file at 1733937824181 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1be97a5d392c877b9bc9b2732b8cfea7 in 458ms, sequenceid=8, compaction requested=false at 1733937824194 (+13 ms)Writing region close event to WAL at 1733937824195 (+1 ms)Running coprocessor post-close hooks at 1733937824200 (+5 ms)Closed at 1733937824200 2024-12-11T17:23:44,201 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733937762638.1be97a5d392c877b9bc9b2732b8cfea7. 2024-12-11T17:23:44,201 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b0d32ac8e8a59c7156e4cf9ea1cb7a23, disabling compactions & flushes 2024-12-11T17:23:44,201 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:44,201 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:44,201 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. after waiting 0 ms 2024-12-11T17:23:44,201 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:44,201 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b0d32ac8e8a59c7156e4cf9ea1cb7a23 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:44,223 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23/.tmp/testFamily/5f2abf60d19f46f8aacd02fdd99d64be is 61, key is row-0/testFamily:testQualifier/1733937801244/Put/seqid=0 2024-12-11T17:23:44,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741863_1039 (size=5049) 2024-12-11T17:23:44,295 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:44,495 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-12-11T17:23:44,495 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1325): Online Regions={b0d32ac8e8a59c7156e4cf9ea1cb7a23=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., 43a788ca58b94f9514910b987039b419=testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., 30d92a4bafdc46c54d1ef5b985214c88=hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88., cb0260ace7c4b7e117e26b221bf6a0d2=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., 88c575eda49795e049781ca96698701c=testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c.} 2024-12-11T17:23:44,495 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 88c575eda49795e049781ca96698701c, b0d32ac8e8a59c7156e4cf9ea1cb7a23, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:44,629 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23/.tmp/testFamily/5f2abf60d19f46f8aacd02fdd99d64be 2024-12-11T17:23:44,638 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23/.tmp/testFamily/5f2abf60d19f46f8aacd02fdd99d64be as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23/testFamily/5f2abf60d19f46f8aacd02fdd99d64be 2024-12-11T17:23:44,648 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23/testFamily/5f2abf60d19f46f8aacd02fdd99d64be, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:44,649 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b0d32ac8e8a59c7156e4cf9ea1cb7a23 in 448ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:44,655 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/b0d32ac8e8a59c7156e4cf9ea1cb7a23/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:44,656 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:44,656 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b0d32ac8e8a59c7156e4cf9ea1cb7a23: Waiting for close lock at 1733937824201Running coprocessor pre-close hooks at 1733937824201Disabling compacts and flushes for region at 1733937824201Disabling writes for close at 1733937824201Obtaining lock to block concurrent updates at 1733937824201Preparing flush snapshotting stores in b0d32ac8e8a59c7156e4cf9ea1cb7a23 at 1733937824201Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937824202 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. at 1733937824203 (+1 ms)Flushing b0d32ac8e8a59c7156e4cf9ea1cb7a23/testFamily: creating writer at 1733937824203Flushing b0d32ac8e8a59c7156e4cf9ea1cb7a23/testFamily: appending metadata at 1733937824222 (+19 ms)Flushing b0d32ac8e8a59c7156e4cf9ea1cb7a23/testFamily: closing flushed file at 1733937824222Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4eb38581: reopening flushed file at 1733937824636 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for b0d32ac8e8a59c7156e4cf9ea1cb7a23 in 448ms, sequenceid=8, compaction requested=false at 1733937824649 (+13 ms)Writing region close event to WAL at 1733937824650 (+1 ms)Running coprocessor post-close hooks at 1733937824656 (+6 ms)Closed at 1733937824656 2024-12-11T17:23:44,657 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733937798196.b0d32ac8e8a59c7156e4cf9ea1cb7a23. 2024-12-11T17:23:44,657 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 43a788ca58b94f9514910b987039b419, disabling compactions & flushes 2024-12-11T17:23:44,657 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:44,657 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:44,657 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. after waiting 0 ms 2024-12-11T17:23:44,657 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:44,657 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 43a788ca58b94f9514910b987039b419 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:44,680 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419/.tmp/testFamily/021e05d63d0e41c6b8dc9d4a50cb60d6 is 61, key is row-0/testFamily:testQualifier/1733937819422/Put/seqid=0 2024-12-11T17:23:44,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741864_1040 (size=5049) 2024-12-11T17:23:44,696 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 88c575eda49795e049781ca96698701c, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:44,896 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 88c575eda49795e049781ca96698701c, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:45,086 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419/.tmp/testFamily/021e05d63d0e41c6b8dc9d4a50cb60d6 2024-12-11T17:23:45,096 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419/.tmp/testFamily/021e05d63d0e41c6b8dc9d4a50cb60d6 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419/testFamily/021e05d63d0e41c6b8dc9d4a50cb60d6 2024-12-11T17:23:45,096 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 30d92a4bafdc46c54d1ef5b985214c88, 43a788ca58b94f9514910b987039b419, 88c575eda49795e049781ca96698701c, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:45,104 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419/testFamily/021e05d63d0e41c6b8dc9d4a50cb60d6, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:45,105 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 43a788ca58b94f9514910b987039b419 in 448ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:45,110 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScannerAsync/43a788ca58b94f9514910b987039b419/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:45,111 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:45,111 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 43a788ca58b94f9514910b987039b419: Waiting for close lock at 1733937824657Running coprocessor pre-close hooks at 1733937824657Disabling compacts and flushes for region at 1733937824657Disabling writes for close at 1733937824657Obtaining lock to block concurrent updates at 1733937824657Preparing flush snapshotting stores in 43a788ca58b94f9514910b987039b419 at 1733937824657Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937824658 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. at 1733937824659 (+1 ms)Flushing 43a788ca58b94f9514910b987039b419/testFamily: creating writer at 1733937824659Flushing 43a788ca58b94f9514910b987039b419/testFamily: appending metadata at 1733937824679 (+20 ms)Flushing 43a788ca58b94f9514910b987039b419/testFamily: closing flushed file at 1733937824680 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19e10dab: reopening flushed file at 1733937825094 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 43a788ca58b94f9514910b987039b419 in 448ms, sequenceid=8, compaction requested=false at 1733937825105 (+11 ms)Writing region close event to WAL at 1733937825106 (+1 ms)Running coprocessor post-close hooks at 1733937825111 (+5 ms)Closed at 1733937825111 2024-12-11T17:23:45,111 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733937816376.43a788ca58b94f9514910b987039b419. 2024-12-11T17:23:45,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30d92a4bafdc46c54d1ef5b985214c88, disabling compactions & flushes 2024-12-11T17:23:45,112 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:23:45,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:23:45,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. after waiting 0 ms 2024-12-11T17:23:45,112 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:23:45,112 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 30d92a4bafdc46c54d1ef5b985214c88 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:45,132 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88/.tmp/testFamily/2bcec15a88d945cbbb9830b6668771f0 is 61, key is row-0/testFamily:testQualifier/1733937774794/Put/seqid=0 2024-12-11T17:23:45,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741865_1041 (size=5049) 2024-12-11T17:23:45,296 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 30d92a4bafdc46c54d1ef5b985214c88, 88c575eda49795e049781ca96698701c, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:45,416 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-11T17:23:45,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T17:23:45,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-11T17:23:45,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-11T17:23:45,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-11T17:23:45,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-11T17:23:45,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-11T17:23:45,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-11T17:23:45,497 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-11T17:23:45,497 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1325): Online Regions={30d92a4bafdc46c54d1ef5b985214c88=hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88., cb0260ace7c4b7e117e26b221bf6a0d2=testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., 88c575eda49795e049781ca96698701c=testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c.} 2024-12-11T17:23:45,497 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 30d92a4bafdc46c54d1ef5b985214c88, 88c575eda49795e049781ca96698701c, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:45,538 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88/.tmp/testFamily/2bcec15a88d945cbbb9830b6668771f0 2024-12-11T17:23:45,547 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88/.tmp/testFamily/2bcec15a88d945cbbb9830b6668771f0 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88/testFamily/2bcec15a88d945cbbb9830b6668771f0 2024-12-11T17:23:45,556 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88/testFamily/2bcec15a88d945cbbb9830b6668771f0, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:45,558 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 30d92a4bafdc46c54d1ef5b985214c88 in 445ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:45,564 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/hbase/testMetaScanTimeoutOnNextAsync/30d92a4bafdc46c54d1ef5b985214c88/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:45,565 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:23:45,565 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30d92a4bafdc46c54d1ef5b985214c88: Waiting for close lock at 1733937825112Running coprocessor pre-close hooks at 1733937825112Disabling compacts and flushes for region at 1733937825112Disabling writes for close at 1733937825112Obtaining lock to block concurrent updates at 1733937825112Preparing flush snapshotting stores in 30d92a4bafdc46c54d1ef5b985214c88 at 1733937825112Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937825112Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. at 1733937825113 (+1 ms)Flushing 30d92a4bafdc46c54d1ef5b985214c88/testFamily: creating writer at 1733937825113Flushing 30d92a4bafdc46c54d1ef5b985214c88/testFamily: appending metadata at 1733937825132 (+19 ms)Flushing 30d92a4bafdc46c54d1ef5b985214c88/testFamily: closing flushed file at 1733937825132Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11952dca: reopening flushed file at 1733937825546 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 30d92a4bafdc46c54d1ef5b985214c88 in 445ms, sequenceid=8, compaction requested=false at 1733937825558 (+12 ms)Writing region close event to WAL at 1733937825559 (+1 ms)Running coprocessor post-close hooks at 1733937825564 (+5 ms)Closed at 1733937825565 (+1 ms) 2024-12-11T17:23:45,565 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733937771753.30d92a4bafdc46c54d1ef5b985214c88. 2024-12-11T17:23:45,565 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cb0260ace7c4b7e117e26b221bf6a0d2, disabling compactions & flushes 2024-12-11T17:23:45,565 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:45,565 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:45,565 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. after waiting 0 ms 2024-12-11T17:23:45,565 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:45,565 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cb0260ace7c4b7e117e26b221bf6a0d2 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:45,580 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2/.tmp/testFamily/fc5b84f0d4484f31a63091a8a8cc30f9 is 61, key is row-0/testFamily:testQualifier/1733937784982/Put/seqid=0 2024-12-11T17:23:45,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741866_1042 (size=5049) 2024-12-11T17:23:45,697 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 88c575eda49795e049781ca96698701c, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:45,897 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 88c575eda49795e049781ca96698701c, cb0260ace7c4b7e117e26b221bf6a0d2 2024-12-11T17:23:45,987 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2/.tmp/testFamily/fc5b84f0d4484f31a63091a8a8cc30f9 2024-12-11T17:23:45,995 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2/.tmp/testFamily/fc5b84f0d4484f31a63091a8a8cc30f9 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2/testFamily/fc5b84f0d4484f31a63091a8a8cc30f9 2024-12-11T17:23:46,002 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2/testFamily/fc5b84f0d4484f31a63091a8a8cc30f9, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:46,003 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cb0260ace7c4b7e117e26b221bf6a0d2 in 438ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:46,008 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testNormalScanTimeoutOnOpenScanner/cb0260ace7c4b7e117e26b221bf6a0d2/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:46,009 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:46,009 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cb0260ace7c4b7e117e26b221bf6a0d2: Waiting for close lock at 1733937825565Running coprocessor pre-close hooks at 1733937825565Disabling compacts and flushes for region at 1733937825565Disabling writes for close at 1733937825565Obtaining lock to block concurrent updates at 1733937825565Preparing flush snapshotting stores in cb0260ace7c4b7e117e26b221bf6a0d2 at 1733937825565Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937825565Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. at 1733937825566 (+1 ms)Flushing cb0260ace7c4b7e117e26b221bf6a0d2/testFamily: creating writer at 1733937825566Flushing cb0260ace7c4b7e117e26b221bf6a0d2/testFamily: appending metadata at 1733937825580 (+14 ms)Flushing cb0260ace7c4b7e117e26b221bf6a0d2/testFamily: closing flushed file at 1733937825580Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ee732b7: reopening flushed file at 1733937825994 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for cb0260ace7c4b7e117e26b221bf6a0d2 in 438ms, sequenceid=8, compaction requested=false at 1733937826003 (+9 ms)Writing region close event to WAL at 1733937826004 (+1 ms)Running coprocessor post-close hooks at 1733937826009 (+5 ms)Closed at 1733937826009 2024-12-11T17:23:46,009 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733937781921.cb0260ace7c4b7e117e26b221bf6a0d2. 2024-12-11T17:23:46,010 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88c575eda49795e049781ca96698701c, disabling compactions & flushes 2024-12-11T17:23:46,010 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:46,010 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:46,010 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. after waiting 0 ms 2024-12-11T17:23:46,010 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:46,010 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 88c575eda49795e049781ca96698701c 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-11T17:23:46,032 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c/.tmp/testFamily/8b9bd49e899a4a939ee44a3e3facc5f6 is 61, key is row-0/testFamily:testQualifier/1733937781863/Put/seqid=0 2024-12-11T17:23:46,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741867_1043 (size=5049) 2024-12-11T17:23:46,098 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 88c575eda49795e049781ca96698701c 2024-12-11T17:23:46,298 DEBUG [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1351): Waiting on 88c575eda49795e049781ca96698701c 2024-12-11T17:23:46,438 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c/.tmp/testFamily/8b9bd49e899a4a939ee44a3e3facc5f6 2024-12-11T17:23:46,446 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c/.tmp/testFamily/8b9bd49e899a4a939ee44a3e3facc5f6 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c/testFamily/8b9bd49e899a4a939ee44a3e3facc5f6 2024-12-11T17:23:46,454 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c/testFamily/8b9bd49e899a4a939ee44a3e3facc5f6, entries=4, sequenceid=8, filesize=4.9 K 2024-12-11T17:23:46,455 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 88c575eda49795e049781ca96698701c in 445ms, sequenceid=8, compaction requested=false 2024-12-11T17:23:46,460 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/data/default/testRetryOutOfOrderScannerNextException/88c575eda49795e049781ca96698701c/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-11T17:23:46,461 INFO [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:46,461 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88c575eda49795e049781ca96698701c: Waiting for close lock at 1733937826010Running coprocessor pre-close hooks at 1733937826010Disabling compacts and flushes for region at 1733937826010Disabling writes for close at 1733937826010Obtaining lock to block concurrent updates at 1733937826010Preparing flush snapshotting stores in 88c575eda49795e049781ca96698701c at 1733937826010Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733937826010Flushing stores of testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. at 1733937826011 (+1 ms)Flushing 88c575eda49795e049781ca96698701c/testFamily: creating writer at 1733937826011Flushing 88c575eda49795e049781ca96698701c/testFamily: appending metadata at 1733937826031 (+20 ms)Flushing 88c575eda49795e049781ca96698701c/testFamily: closing flushed file at 1733937826031Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10622414: reopening flushed file at 1733937826445 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 88c575eda49795e049781ca96698701c in 445ms, sequenceid=8, compaction requested=false at 1733937826455 (+10 ms)Writing region close event to WAL at 1733937826456 (+1 ms)Running coprocessor post-close hooks at 1733937826461 (+5 ms)Closed at 1733937826461 2024-12-11T17:23:46,461 DEBUG [RS_CLOSE_REGION-regionserver/790ba370077b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733937780840.88c575eda49795e049781ca96698701c. 2024-12-11T17:23:46,498 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(976): stopping server 790ba370077b,39119,1733937725853; all regions closed. 2024-12-11T17:23:46,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741834_1010 (size=19479) 2024-12-11T17:23:46,508 DEBUG [RS:0;790ba370077b:39119 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/oldWALs 2024-12-11T17:23:46,508 INFO [RS:0;790ba370077b:39119 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 790ba370077b%2C39119%2C1733937725853.meta:.meta(num 1733937758980) 2024-12-11T17:23:46,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741833_1009 (size=20031) 2024-12-11T17:23:46,511 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/WALs/790ba370077b,39119,1733937725853/790ba370077b%2C39119%2C1733937725853.1733937758441 not finished, retry = 0 2024-12-11T17:23:46,615 DEBUG [RS:0;790ba370077b:39119 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/oldWALs 2024-12-11T17:23:46,615 INFO [RS:0;790ba370077b:39119 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 790ba370077b%2C39119%2C1733937725853:(num 1733937758441) 2024-12-11T17:23:46,615 DEBUG [RS:0;790ba370077b:39119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T17:23:46,615 INFO [RS:0;790ba370077b:39119 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T17:23:46,615 INFO [RS:0;790ba370077b:39119 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T17:23:46,615 INFO [RS:0;790ba370077b:39119 {}] hbase.ChoreService(370): Chore service for: regionserver/790ba370077b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-11T17:23:46,616 INFO [RS:0;790ba370077b:39119 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T17:23:46,616 INFO [regionserver/790ba370077b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T17:23:46,617 INFO [RS:0;790ba370077b:39119 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39119 2024-12-11T17:23:46,669 INFO [regionserver/790ba370077b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T17:23:46,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T17:23:46,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/790ba370077b,39119,1733937725853 2024-12-11T17:23:46,684 INFO [RS:0;790ba370077b:39119 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T17:23:46,685 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [790ba370077b,39119,1733937725853] 2024-12-11T17:23:46,702 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/790ba370077b,39119,1733937725853 already deleted, retry=false 2024-12-11T17:23:46,702 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 790ba370077b,39119,1733937725853 expired; onlineServers=0 2024-12-11T17:23:46,702 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '790ba370077b,38569,1733937724982' ***** 2024-12-11T17:23:46,702 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-11T17:23:46,702 INFO [M:0;790ba370077b:38569 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T17:23:46,702 INFO [M:0;790ba370077b:38569 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T17:23:46,702 DEBUG [M:0;790ba370077b:38569 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-11T17:23:46,703 DEBUG [M:0;790ba370077b:38569 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-11T17:23:46,703 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-11T17:23:46,703 DEBUG [master/790ba370077b:0:becomeActiveMaster-HFileCleaner.large.0-1733937728220 {}] cleaner.HFileCleaner(306): Exit Thread[master/790ba370077b:0:becomeActiveMaster-HFileCleaner.large.0-1733937728220,5,FailOnTimeoutGroup] 2024-12-11T17:23:46,703 DEBUG [master/790ba370077b:0:becomeActiveMaster-HFileCleaner.small.0-1733937728221 {}] cleaner.HFileCleaner(306): Exit Thread[master/790ba370077b:0:becomeActiveMaster-HFileCleaner.small.0-1733937728221,5,FailOnTimeoutGroup] 2024-12-11T17:23:46,703 INFO [M:0;790ba370077b:38569 {}] hbase.ChoreService(370): Chore service for: master/790ba370077b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-11T17:23:46,703 INFO [M:0;790ba370077b:38569 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T17:23:46,703 DEBUG [M:0;790ba370077b:38569 {}] master.HMaster(1795): Stopping service threads 2024-12-11T17:23:46,703 INFO [M:0;790ba370077b:38569 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-11T17:23:46,703 INFO [M:0;790ba370077b:38569 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T17:23:46,704 INFO [M:0;790ba370077b:38569 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T17:23:46,704 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-11T17:23:46,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-11T17:23:46,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T17:23:46,710 DEBUG [M:0;790ba370077b:38569 {}] zookeeper.ZKUtil(347): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-11T17:23:46,710 WARN [M:0;790ba370077b:38569 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-11T17:23:46,711 INFO [M:0;790ba370077b:38569 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/.lastflushedseqids 2024-12-11T17:23:46,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741868_1044 (size=721) 2024-12-11T17:23:46,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T17:23:46,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39119-0x10016012ad60001, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T17:23:46,794 INFO [RS:0;790ba370077b:39119 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T17:23:46,795 INFO [RS:0;790ba370077b:39119 {}] regionserver.HRegionServer(1031): Exiting; stopping=790ba370077b,39119,1733937725853; zookeeper connection closed. 2024-12-11T17:23:46,795 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@689b411d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@689b411d 2024-12-11T17:23:46,795 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-11T17:23:47,121 INFO [M:0;790ba370077b:38569 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T17:23:47,122 INFO [M:0;790ba370077b:38569 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-11T17:23:47,122 DEBUG [M:0;790ba370077b:38569 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T17:23:47,122 INFO [M:0;790ba370077b:38569 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T17:23:47,122 DEBUG [M:0;790ba370077b:38569 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T17:23:47,122 DEBUG [M:0;790ba370077b:38569 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T17:23:47,122 DEBUG [M:0;790ba370077b:38569 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T17:23:47,122 INFO [M:0;790ba370077b:38569 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.04 KB heapSize=191.06 KB 2024-12-11T17:23:47,144 DEBUG [M:0;790ba370077b:38569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2bc1b1ee8ae6467c82452fff8c653d82 is 82, key is hbase:meta,,1/info:regioninfo/1733937759069/Put/seqid=0 2024-12-11T17:23:47,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741869_1045 (size=5672) 2024-12-11T17:23:47,550 INFO [M:0;790ba370077b:38569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2bc1b1ee8ae6467c82452fff8c653d82 2024-12-11T17:23:47,582 DEBUG [M:0;790ba370077b:38569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71f3c709adf74a4abc20855afc697c00 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733937760698/Put/seqid=0 2024-12-11T17:23:47,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741870_1046 (size=14066) 2024-12-11T17:23:47,595 INFO [M:0;790ba370077b:38569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.49 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71f3c709adf74a4abc20855afc697c00 2024-12-11T17:23:47,603 INFO [M:0;790ba370077b:38569 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 71f3c709adf74a4abc20855afc697c00 2024-12-11T17:23:47,625 DEBUG [M:0;790ba370077b:38569 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0aedb87ed4624631a29d685ee382cb96 is 69, key is 790ba370077b,39119,1733937725853/rs:state/1733937728279/Put/seqid=0 2024-12-11T17:23:47,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741871_1047 (size=5156) 2024-12-11T17:23:48,355 INFO [M:0;790ba370077b:38569 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0aedb87ed4624631a29d685ee382cb96 2024-12-11T17:23:48,363 DEBUG [M:0;790ba370077b:38569 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2bc1b1ee8ae6467c82452fff8c653d82 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2bc1b1ee8ae6467c82452fff8c653d82 2024-12-11T17:23:48,370 INFO [M:0;790ba370077b:38569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2bc1b1ee8ae6467c82452fff8c653d82, entries=8, sequenceid=319, filesize=5.5 K 2024-12-11T17:23:48,371 DEBUG [M:0;790ba370077b:38569 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/71f3c709adf74a4abc20855afc697c00 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/71f3c709adf74a4abc20855afc697c00 2024-12-11T17:23:48,378 INFO [M:0;790ba370077b:38569 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 71f3c709adf74a4abc20855afc697c00 2024-12-11T17:23:48,379 INFO [M:0;790ba370077b:38569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/71f3c709adf74a4abc20855afc697c00, entries=33, sequenceid=319, filesize=13.7 K 2024-12-11T17:23:48,380 DEBUG [M:0;790ba370077b:38569 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0aedb87ed4624631a29d685ee382cb96 as hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0aedb87ed4624631a29d685ee382cb96 2024-12-11T17:23:48,388 INFO [M:0;790ba370077b:38569 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34667/user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0aedb87ed4624631a29d685ee382cb96, entries=1, sequenceid=319, filesize=5.0 K 2024-12-11T17:23:48,389 INFO [M:0;790ba370077b:38569 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.04 KB/166955, heapSize ~190.77 KB/195344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1267ms, sequenceid=319, compaction requested=false 2024-12-11T17:23:48,392 INFO [M:0;790ba370077b:38569 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T17:23:48,392 DEBUG [M:0;790ba370077b:38569 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733937827122Disabling compacts and flushes for region at 1733937827122Disabling writes for close at 1733937827122Obtaining lock to block concurrent updates at 1733937827122Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733937827122Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166955, getHeapSize=195584, getOffHeapSize=0, getCellsCount=366 at 1733937827122Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733937827123 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733937827123Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733937827144 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733937827144Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733937827558 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733937827581 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733937827581Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733937827603 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733937827624 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733937827624Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65f5943: reopening flushed file at 1733937828362 (+738 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@778d4992: reopening flushed file at 1733937828370 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ad256ba: reopening flushed file at 1733937828379 (+9 ms)Finished flush of dataSize ~163.04 KB/166955, heapSize ~190.77 KB/195344, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1267ms, sequenceid=319, compaction requested=false at 1733937828389 (+10 ms)Writing region close event to WAL at 1733937828392 (+3 ms)Closed at 1733937828392 2024-12-11T17:23:48,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34293 is added to blk_1073741830_1006 (size=188297) 2024-12-11T17:23:48,395 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/566c5b1e-2306-1632-e5c2-d1fdc7c95d90/MasterData/WALs/790ba370077b,38569,1733937724982/790ba370077b%2C38569%2C1733937724982.1733937727387 not finished, retry = 0 2024-12-11T17:23:48,497 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T17:23:48,497 INFO [M:0;790ba370077b:38569 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-11T17:23:48,497 INFO [M:0;790ba370077b:38569 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38569 2024-12-11T17:23:48,498 INFO [M:0;790ba370077b:38569 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T17:23:48,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T17:23:48,620 INFO [M:0;790ba370077b:38569 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T17:23:48,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38569-0x10016012ad60000, quorum=127.0.0.1:63961, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T17:23:48,625 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T17:23:48,627 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T17:23:48,627 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T17:23:48,628 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T17:23:48,628 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/hadoop.log.dir/,STOPPED} 2024-12-11T17:23:48,633 WARN [BP-1757774003-172.17.0.3-1733937721344 heartbeating to localhost/127.0.0.1:34667 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-11T17:23:48,633 WARN [BP-1757774003-172.17.0.3-1733937721344 heartbeating to localhost/127.0.0.1:34667 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1757774003-172.17.0.3-1733937721344 (Datanode Uuid 40bc0b08-6cc9-448a-bdd3-6dd27ea06d79) service to localhost/127.0.0.1:34667 2024-12-11T17:23:48,634 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-11T17:23:48,634 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-11T17:23:48,634 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/cluster_113cb4fb-a821-384d-7ca6-aec806970df0/data/data1/current/BP-1757774003-172.17.0.3-1733937721344 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T17:23:48,635 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/cluster_113cb4fb-a821-384d-7ca6-aec806970df0/data/data2/current/BP-1757774003-172.17.0.3-1733937721344 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T17:23:48,635 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-11T17:23:48,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T17:23:48,647 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T17:23:48,647 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T17:23:48,648 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T17:23:48,648 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/250e3861-8e97-0f28-b333-65ce6d902c1c/hadoop.log.dir/,STOPPED} 2024-12-11T17:23:48,657 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-11T17:23:48,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down