2024-12-03 10:22:10,061 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 10:22:10,078 main DEBUG Took 0.014526 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 10:22:10,078 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 10:22:10,079 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 10:22:10,080 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 10:22:10,082 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,091 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 10:22:10,120 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,123 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,123 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,124 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,125 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,126 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,128 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,129 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,129 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,130 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,130 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,131 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,131 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,132 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,133 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,133 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,134 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,134 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 10:22:10,135 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,135 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 10:22:10,138 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 10:22:10,139 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 10:22:10,142 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 10:22:10,143 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 10:22:10,145 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 10:22:10,146 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 10:22:10,157 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 10:22:10,160 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 10:22:10,163 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 10:22:10,163 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 10:22:10,164 main DEBUG createAppenders(={Console}) 2024-12-03 10:22:10,165 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-12-03 10:22:10,166 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-12-03 10:22:10,166 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-12-03 10:22:10,167 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 10:22:10,167 main DEBUG OutputStream closed 2024-12-03 10:22:10,168 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 10:22:10,168 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 10:22:10,169 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-12-03 10:22:10,278 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 10:22:10,281 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 10:22:10,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 10:22:10,283 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 10:22:10,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 10:22:10,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 10:22:10,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 10:22:10,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 10:22:10,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 10:22:10,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 10:22:10,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 10:22:10,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 10:22:10,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 10:22:10,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 10:22:10,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 10:22:10,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 10:22:10,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 10:22:10,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 10:22:10,294 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 10:22:10,295 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-12-03 10:22:10,295 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 10:22:10,296 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-12-03T10:22:10,315 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-12-03 10:22:10,319 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 10:22:10,320 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T10:22:10,648 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719 2024-12-03T10:22:10,680 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T10:22:10,698 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/cluster_9091d149-19d1-21ab-6965-e2071c9733a5, deleteOnExit=true 2024-12-03T10:22:10,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T10:22:10,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/test.cache.data in system properties and HBase conf 2024-12-03T10:22:10,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T10:22:10,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/hadoop.log.dir in system properties and HBase conf 2024-12-03T10:22:10,702 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T10:22:10,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T10:22:10,703 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T10:22:10,820 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T10:22:10,934 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T10:22:10,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T10:22:10,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T10:22:10,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T10:22:10,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T10:22:10,941 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T10:22:10,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T10:22:10,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T10:22:10,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T10:22:10,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T10:22:10,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/nfs.dump.dir in system properties and HBase conf 2024-12-03T10:22:10,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/java.io.tmpdir in system properties and HBase conf 2024-12-03T10:22:10,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T10:22:10,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T10:22:10,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T10:22:11,925 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T10:22:11,998 INFO [Time-limited test {}] log.Log(170): Logging initialized @2723ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T10:22:12,075 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T10:22:12,129 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T10:22:12,145 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T10:22:12,146 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T10:22:12,147 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T10:22:12,158 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T10:22:12,160 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72daf39c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/hadoop.log.dir/,AVAILABLE} 2024-12-03T10:22:12,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13f713ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T10:22:12,324 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21a9f2dc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/java.io.tmpdir/jetty-localhost-37505-hadoop-hdfs-3_4_1-tests_jar-_-any-16434394617407082504/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T10:22:12,330 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b7efa4f{HTTP/1.1, (http/1.1)}{localhost:37505} 2024-12-03T10:22:12,331 INFO [Time-limited test {}] server.Server(415): Started @3056ms 2024-12-03T10:22:12,808 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T10:22:12,817 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T10:22:12,818 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T10:22:12,818 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T10:22:12,819 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T10:22:12,820 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39e2c913{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/hadoop.log.dir/,AVAILABLE} 2024-12-03T10:22:12,820 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1cc8666a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T10:22:12,915 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5589cdf5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/java.io.tmpdir/jetty-localhost-43799-hadoop-hdfs-3_4_1-tests_jar-_-any-14103060709574731989/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T10:22:12,915 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@75f79e6f{HTTP/1.1, (http/1.1)}{localhost:43799} 2024-12-03T10:22:12,915 INFO [Time-limited test {}] server.Server(415): Started @3641ms 2024-12-03T10:22:12,964 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T10:22:13,991 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/cluster_9091d149-19d1-21ab-6965-e2071c9733a5/data/data2/current/BP-953270253-172.17.0.2-1733221331514/current, will proceed with Du for space computation calculation, 2024-12-03T10:22:13,991 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/cluster_9091d149-19d1-21ab-6965-e2071c9733a5/data/data1/current/BP-953270253-172.17.0.2-1733221331514/current, will proceed with Du for space computation calculation, 2024-12-03T10:22:14,018 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T10:22:14,062 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25491e81103bb21c with lease ID 0xbd80e7407bc55ca6: Processing first storage report for DS-077d0e77-0915-417f-ae33-a9a595bb623b from datanode DatanodeRegistration(127.0.0.1:35377, datanodeUuid=144d94f9-5688-48b7-b8c6-5c34939ecb05, infoPort=44707, infoSecurePort=0, ipcPort=43071, storageInfo=lv=-57;cid=testClusterID;nsid=685291625;c=1733221331514) 2024-12-03T10:22:14,064 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25491e81103bb21c with lease ID 0xbd80e7407bc55ca6: from storage DS-077d0e77-0915-417f-ae33-a9a595bb623b node DatanodeRegistration(127.0.0.1:35377, datanodeUuid=144d94f9-5688-48b7-b8c6-5c34939ecb05, infoPort=44707, infoSecurePort=0, ipcPort=43071, storageInfo=lv=-57;cid=testClusterID;nsid=685291625;c=1733221331514), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T10:22:14,064 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x25491e81103bb21c with lease ID 0xbd80e7407bc55ca6: Processing first storage report for DS-c12700a8-291e-4389-972b-c5025c0895cc from datanode DatanodeRegistration(127.0.0.1:35377, datanodeUuid=144d94f9-5688-48b7-b8c6-5c34939ecb05, infoPort=44707, infoSecurePort=0, ipcPort=43071, storageInfo=lv=-57;cid=testClusterID;nsid=685291625;c=1733221331514) 2024-12-03T10:22:14,065 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x25491e81103bb21c with lease ID 0xbd80e7407bc55ca6: from storage DS-c12700a8-291e-4389-972b-c5025c0895cc node DatanodeRegistration(127.0.0.1:35377, datanodeUuid=144d94f9-5688-48b7-b8c6-5c34939ecb05, infoPort=44707, infoSecurePort=0, ipcPort=43071, storageInfo=lv=-57;cid=testClusterID;nsid=685291625;c=1733221331514), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T10:22:14,091 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719 2024-12-03T10:22:14,157 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/cluster_9091d149-19d1-21ab-6965-e2071c9733a5/zookeeper_0, clientPort=55781, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/cluster_9091d149-19d1-21ab-6965-e2071c9733a5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/cluster_9091d149-19d1-21ab-6965-e2071c9733a5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T10:22:14,167 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55781 2024-12-03T10:22:14,180 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T10:22:14,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T10:22:14,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741825_1001 (size=7) 2024-12-03T10:22:14,800 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 with version=8 2024-12-03T10:22:14,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/hbase-staging 2024-12-03T10:22:14,872 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T10:22:15,094 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a78f5601770a:0 server-side Connection retries=9 2024-12-03T10:22:15,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T10:22:15,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T10:22:15,106 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T10:22:15,107 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T10:22:15,107 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T10:22:15,233 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T10:22:15,310 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T10:22:15,319 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T10:22:15,323 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T10:22:15,352 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 73410 (auto-detected) 2024-12-03T10:22:15,353 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T10:22:15,378 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36731 2024-12-03T10:22:15,402 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36731 connecting to ZooKeeper ensemble=127.0.0.1:55781 2024-12-03T10:22:15,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:367310x0, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T10:22:15,511 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36731-0x1019c006e650000 connected 2024-12-03T10:22:15,605 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T10:22:15,607 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T10:22:15,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T10:22:15,625 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2, hbase.cluster.distributed=false 2024-12-03T10:22:15,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T10:22:15,664 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36731 2024-12-03T10:22:15,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36731 2024-12-03T10:22:15,666 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36731 2024-12-03T10:22:15,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36731 2024-12-03T10:22:15,671 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36731 2024-12-03T10:22:15,795 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a78f5601770a:0 server-side Connection retries=9 2024-12-03T10:22:15,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T10:22:15,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T10:22:15,797 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T10:22:15,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T10:22:15,797 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T10:22:15,800 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T10:22:15,802 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T10:22:15,802 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38085 2024-12-03T10:22:15,804 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38085 connecting to ZooKeeper ensemble=127.0.0.1:55781 2024-12-03T10:22:15,805 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T10:22:15,809 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T10:22:15,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380850x0, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T10:22:15,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:380850x0, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T10:22:15,825 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38085-0x1019c006e650001 connected 2024-12-03T10:22:15,828 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T10:22:15,834 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T10:22:15,837 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T10:22:15,841 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T10:22:15,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38085 2024-12-03T10:22:15,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38085 2024-12-03T10:22:15,842 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38085 2024-12-03T10:22:15,843 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38085 2024-12-03T10:22:15,845 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38085 2024-12-03T10:22:15,860 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a78f5601770a:36731 2024-12-03T10:22:15,860 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a78f5601770a,36731,1733221334944 2024-12-03T10:22:15,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T10:22:15,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T10:22:15,876 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a78f5601770a,36731,1733221334944 2024-12-03T10:22:15,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T10:22:15,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:15,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:15,904 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T10:22:15,905 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a78f5601770a,36731,1733221334944 from backup master directory 2024-12-03T10:22:15,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a78f5601770a,36731,1733221334944 2024-12-03T10:22:15,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T10:22:15,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T10:22:15,916 WARN [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T10:22:15,916 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a78f5601770a,36731,1733221334944 2024-12-03T10:22:15,918 INFO [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T10:22:15,920 INFO [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T10:22:15,973 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/hbase.id] with ID: 6d730235-4285-4bc0-a468-a95c428290a0 2024-12-03T10:22:15,973 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/.tmp/hbase.id 2024-12-03T10:22:15,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741826_1002 (size=42) 2024-12-03T10:22:16,387 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/.tmp/hbase.id]:[hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/hbase.id] 2024-12-03T10:22:16,434 INFO [master/a78f5601770a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T10:22:16,440 INFO [master/a78f5601770a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T10:22:16,457 INFO [master/a78f5601770a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-03T10:22:16,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:16,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:16,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741827_1003 (size=196) 2024-12-03T10:22:16,925 INFO [master/a78f5601770a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:22:16,927 INFO [master/a78f5601770a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T10:22:16,941 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T10:22:16,945 INFO [master/a78f5601770a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T10:22:16,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741828_1004 (size=1189) 2024-12-03T10:22:16,989 INFO [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store 2024-12-03T10:22:17,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741829_1005 (size=34) 2024-12-03T10:22:17,420 INFO [master/a78f5601770a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T10:22:17,423 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:22:17,424 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T10:22:17,424 INFO [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T10:22:17,425 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T10:22:17,426 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T10:22:17,426 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T10:22:17,426 INFO [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T10:22:17,427 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733221337424Disabling compacts and flushes for region at 1733221337424Disabling writes for close at 1733221337426 (+2 ms)Writing region close event to WAL at 1733221337426Closed at 1733221337426 2024-12-03T10:22:17,430 WARN [master/a78f5601770a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/.initializing 2024-12-03T10:22:17,430 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/WALs/a78f5601770a,36731,1733221334944 2024-12-03T10:22:17,438 INFO [master/a78f5601770a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T10:22:17,452 INFO [master/a78f5601770a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a78f5601770a%2C36731%2C1733221334944, suffix=, logDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/WALs/a78f5601770a,36731,1733221334944, archiveDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/oldWALs, maxLogs=10 2024-12-03T10:22:17,471 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/WALs/a78f5601770a,36731,1733221334944/a78f5601770a%2C36731%2C1733221334944.1733221337456, exclude list is [], retry=0 2024-12-03T10:22:17,486 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35377,DS-077d0e77-0915-417f-ae33-a9a595bb623b,DISK] 2024-12-03T10:22:17,488 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T10:22:17,519 INFO [master/a78f5601770a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/WALs/a78f5601770a,36731,1733221334944/a78f5601770a%2C36731%2C1733221334944.1733221337456 2024-12-03T10:22:17,520 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44707:44707)] 2024-12-03T10:22:17,521 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:22:17,521 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:22:17,524 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,525 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,578 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T10:22:17,581 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:17,583 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:17,584 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,587 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T10:22:17,587 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:17,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:22:17,588 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,591 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T10:22:17,591 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:17,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:22:17,592 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T10:22:17,595 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:17,596 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:22:17,597 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,600 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,601 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,607 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,608 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,612 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T10:22:17,616 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T10:22:17,620 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:22:17,621 INFO [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65490018, jitterRate=-0.024122685194015503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T10:22:17,626 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733221337536Initializing all the Stores at 1733221337538 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733221337538Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221337539 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221337539Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221337539Cleaning up temporary data from old regions at 1733221337608 (+69 ms)Region opened successfully at 1733221337626 (+18 ms) 2024-12-03T10:22:17,627 INFO [master/a78f5601770a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T10:22:17,658 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443dbe33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a78f5601770a/172.17.0.2:0 2024-12-03T10:22:17,686 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T10:22:17,696 INFO [master/a78f5601770a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T10:22:17,697 INFO [master/a78f5601770a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T10:22:17,699 INFO [master/a78f5601770a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T10:22:17,701 INFO [master/a78f5601770a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T10:22:17,705 INFO [master/a78f5601770a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-03T10:22:17,706 INFO [master/a78f5601770a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T10:22:17,729 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T10:22:17,737 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T10:22:17,769 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T10:22:17,772 INFO [master/a78f5601770a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T10:22:17,773 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T10:22:17,781 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T10:22:17,784 INFO [master/a78f5601770a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T10:22:17,787 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T10:22:17,798 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T10:22:17,800 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T10:22:17,811 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T10:22:17,827 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T10:22:17,836 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T10:22:17,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T10:22:17,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T10:22:17,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:17,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:17,853 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a78f5601770a,36731,1733221334944, sessionid=0x1019c006e650000, setting cluster-up flag (Was=false) 2024-12-03T10:22:17,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:17,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:17,907 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T10:22:17,911 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a78f5601770a,36731,1733221334944 2024-12-03T10:22:17,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:17,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:17,965 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T10:22:17,969 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a78f5601770a,36731,1733221334944 2024-12-03T10:22:17,978 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T10:22:18,048 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T10:22:18,051 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(746): ClusterId : 6d730235-4285-4bc0-a468-a95c428290a0 2024-12-03T10:22:18,053 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T10:22:18,057 INFO [master/a78f5601770a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T10:22:18,063 INFO [master/a78f5601770a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T10:22:18,067 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T10:22:18,067 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T10:22:18,069 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a78f5601770a,36731,1733221334944 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T10:22:18,079 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T10:22:18,080 DEBUG [RS:0;a78f5601770a:38085 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a8e993b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a78f5601770a/172.17.0.2:0 2024-12-03T10:22:18,080 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a78f5601770a:0, corePoolSize=5, maxPoolSize=5 2024-12-03T10:22:18,080 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a78f5601770a:0, corePoolSize=5, maxPoolSize=5 2024-12-03T10:22:18,081 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a78f5601770a:0, corePoolSize=5, maxPoolSize=5 2024-12-03T10:22:18,081 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a78f5601770a:0, corePoolSize=5, maxPoolSize=5 2024-12-03T10:22:18,081 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a78f5601770a:0, corePoolSize=10, maxPoolSize=10 2024-12-03T10:22:18,082 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,082 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a78f5601770a:0, corePoolSize=2, maxPoolSize=2 2024-12-03T10:22:18,082 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,087 INFO [master/a78f5601770a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733221368087 2024-12-03T10:22:18,088 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T10:22:18,088 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T10:22:18,088 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T10:22:18,089 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T10:22:18,092 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T10:22:18,093 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T10:22:18,093 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T10:22:18,093 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T10:22:18,094 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:18,095 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T10:22:18,094 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,097 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a78f5601770a:38085 2024-12-03T10:22:18,098 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T10:22:18,100 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T10:22:18,100 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T10:22:18,101 INFO [RS:0;a78f5601770a:38085 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T10:22:18,101 INFO [RS:0;a78f5601770a:38085 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T10:22:18,101 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T10:22:18,102 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T10:22:18,103 INFO [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T10:22:18,104 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(2659): reportForDuty to master=a78f5601770a,36731,1733221334944 with port=38085, startcode=1733221335758 2024-12-03T10:22:18,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741831_1007 (size=1321) 2024-12-03T10:22:18,105 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a78f5601770a:0:becomeActiveMaster-HFileCleaner.large.0-1733221338104,5,FailOnTimeoutGroup] 2024-12-03T10:22:18,105 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a78f5601770a:0:becomeActiveMaster-HFileCleaner.small.0-1733221338105,5,FailOnTimeoutGroup] 2024-12-03T10:22:18,105 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,106 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T10:22:18,107 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,107 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,113 DEBUG [RS:0;a78f5601770a:38085 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T10:22:18,142 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43039, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T10:22:18,147 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36731 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a78f5601770a,38085,1733221335758 2024-12-03T10:22:18,149 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36731 {}] master.ServerManager(517): Registering regionserver=a78f5601770a,38085,1733221335758 2024-12-03T10:22:18,161 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:22:18,161 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35783 2024-12-03T10:22:18,161 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T10:22:18,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T10:22:18,174 DEBUG [RS:0;a78f5601770a:38085 {}] zookeeper.ZKUtil(111): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a78f5601770a,38085,1733221335758 2024-12-03T10:22:18,174 WARN [RS:0;a78f5601770a:38085 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T10:22:18,174 INFO [RS:0;a78f5601770a:38085 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T10:22:18,175 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/WALs/a78f5601770a,38085,1733221335758 2024-12-03T10:22:18,177 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a78f5601770a,38085,1733221335758] 2024-12-03T10:22:18,204 INFO [RS:0;a78f5601770a:38085 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T10:22:18,215 INFO [RS:0;a78f5601770a:38085 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T10:22:18,219 INFO [RS:0;a78f5601770a:38085 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T10:22:18,219 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,220 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T10:22:18,225 INFO [RS:0;a78f5601770a:38085 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T10:22:18,227 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,227 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,227 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,228 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,228 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,228 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,228 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a78f5601770a:0, corePoolSize=2, maxPoolSize=2 2024-12-03T10:22:18,228 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,228 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,229 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,229 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,229 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,229 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a78f5601770a:0, corePoolSize=1, maxPoolSize=1 2024-12-03T10:22:18,229 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a78f5601770a:0, corePoolSize=3, maxPoolSize=3 2024-12-03T10:22:18,230 DEBUG [RS:0;a78f5601770a:38085 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a78f5601770a:0, corePoolSize=3, maxPoolSize=3 2024-12-03T10:22:18,231 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,231 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,231 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,231 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,231 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,231 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=a78f5601770a,38085,1733221335758-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T10:22:18,250 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T10:22:18,251 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=a78f5601770a,38085,1733221335758-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,252 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,252 INFO [RS:0;a78f5601770a:38085 {}] regionserver.Replication(171): a78f5601770a,38085,1733221335758 started 2024-12-03T10:22:18,272 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:18,272 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1482): Serving as a78f5601770a,38085,1733221335758, RpcServer on a78f5601770a/172.17.0.2:38085, sessionid=0x1019c006e650001 2024-12-03T10:22:18,273 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T10:22:18,273 DEBUG [RS:0;a78f5601770a:38085 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a78f5601770a,38085,1733221335758 2024-12-03T10:22:18,273 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a78f5601770a,38085,1733221335758' 2024-12-03T10:22:18,273 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T10:22:18,274 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T10:22:18,275 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T10:22:18,275 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T10:22:18,275 DEBUG [RS:0;a78f5601770a:38085 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a78f5601770a,38085,1733221335758 2024-12-03T10:22:18,275 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a78f5601770a,38085,1733221335758' 2024-12-03T10:22:18,275 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T10:22:18,276 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T10:22:18,277 DEBUG [RS:0;a78f5601770a:38085 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T10:22:18,277 INFO [RS:0;a78f5601770a:38085 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T10:22:18,277 INFO [RS:0;a78f5601770a:38085 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T10:22:18,510 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T10:22:18,511 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:22:18,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741832_1008 (size=32) 2024-12-03T10:22:18,930 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:22:18,935 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T10:22:18,938 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T10:22:18,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:18,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:18,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T10:22:18,942 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T10:22:18,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:18,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:18,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T10:22:18,946 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T10:22:18,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:18,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:18,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T10:22:18,950 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T10:22:18,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:18,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:18,952 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T10:22:18,954 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740 2024-12-03T10:22:18,955 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740 2024-12-03T10:22:18,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T10:22:18,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T10:22:18,959 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T10:22:18,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T10:22:18,966 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:22:18,967 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61837443, jitterRate=-0.0785502940416336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T10:22:18,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733221338931Initializing all the Stores at 1733221338934 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733221338934Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733221338934Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221338935 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733221338935Cleaning up temporary data from old regions at 1733221338958 (+23 ms)Region opened successfully at 1733221338968 (+10 ms) 2024-12-03T10:22:18,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T10:22:18,969 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T10:22:18,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T10:22:18,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T10:22:18,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T10:22:18,970 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T10:22:18,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733221338969Disabling compacts and flushes for region at 1733221338969Disabling writes for close at 1733221338969Writing region close event to WAL at 1733221338970 (+1 ms)Closed at 1733221338970 2024-12-03T10:22:18,973 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T10:22:18,973 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T10:22:18,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T10:22:18,985 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T10:22:18,988 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T10:22:19,140 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:20,141 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:21,142 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:22,143 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:23,143 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:24,144 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:25,145 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:25,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T10:22:25,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T10:22:26,145 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:27,146 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:28,147 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:29,148 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:30,149 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:31,150 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:32,151 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:33,152 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:34,152 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:35,153 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:36,154 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:37,155 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:38,156 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:39,157 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:40,158 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:41,160 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:42,160 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:43,161 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:44,089 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T10:22:44,162 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:45,163 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:46,164 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:47,165 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:48,166 WARN [a78f5601770a:36731 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T10:22:48,291 INFO [RS:0;a78f5601770a:38085 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T10:22:48,295 INFO [RS:0;a78f5601770a:38085 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a78f5601770a%2C38085%2C1733221335758, suffix=, logDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/WALs/a78f5601770a,38085,1733221335758, archiveDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/oldWALs, maxLogs=32 2024-12-03T10:22:48,308 DEBUG [RS:0;a78f5601770a:38085 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/WALs/a78f5601770a,38085,1733221335758/a78f5601770a%2C38085%2C1733221335758.1733221368296, exclude list is [], retry=0 2024-12-03T10:22:48,312 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35377,DS-077d0e77-0915-417f-ae33-a9a595bb623b,DISK] 2024-12-03T10:22:48,315 INFO [RS:0;a78f5601770a:38085 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/WALs/a78f5601770a,38085,1733221335758/a78f5601770a%2C38085%2C1733221335758.1733221368296 2024-12-03T10:22:48,315 DEBUG [RS:0;a78f5601770a:38085 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44707:44707)] 2024-12-03T10:22:48,419 DEBUG [a78f5601770a:36731 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-03T10:22:48,434 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:22:48,441 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a78f5601770a,38085,1733221335758, state=OPENING 2024-12-03T10:22:48,471 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T10:22:48,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:48,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:22:48,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T10:22:48,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T10:22:48,485 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T10:22:48,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:22:48,667 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T10:22:48,670 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41399, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T10:22:48,681 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T10:22:48,681 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T10:22:48,682 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T10:22:48,686 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a78f5601770a%2C38085%2C1733221335758.meta, suffix=.meta, logDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/WALs/a78f5601770a,38085,1733221335758, archiveDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/oldWALs, maxLogs=32 2024-12-03T10:22:48,700 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/WALs/a78f5601770a,38085,1733221335758/a78f5601770a%2C38085%2C1733221335758.meta.1733221368688.meta, exclude list is [], retry=0 2024-12-03T10:22:48,704 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35377,DS-077d0e77-0915-417f-ae33-a9a595bb623b,DISK] 2024-12-03T10:22:48,707 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/WALs/a78f5601770a,38085,1733221335758/a78f5601770a%2C38085%2C1733221335758.meta.1733221368688.meta 2024-12-03T10:22:48,707 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44707:44707)] 2024-12-03T10:22:48,708 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:22:48,709 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T10:22:48,711 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T10:22:48,715 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T10:22:48,719 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T10:22:48,720 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:22:48,720 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T10:22:48,720 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T10:22:48,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T10:22:48,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T10:22:48,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:48,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:48,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T10:22:48,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T10:22:48,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:48,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:48,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T10:22:48,731 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T10:22:48,731 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:48,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:48,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T10:22:48,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T10:22:48,734 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:48,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T10:22:48,735 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T10:22:48,736 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740 2024-12-03T10:22:48,740 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740 2024-12-03T10:22:48,743 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T10:22:48,743 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T10:22:48,744 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T10:22:48,747 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T10:22:48,748 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62008192, jitterRate=-0.07600593566894531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T10:22:48,749 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T10:22:48,750 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733221368721Writing region info on filesystem at 1733221368721Initializing all the Stores at 1733221368723 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733221368723Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733221368724 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221368724Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733221368724Cleaning up temporary data from old regions at 1733221368743 (+19 ms)Running coprocessor post-open hooks at 1733221368749 (+6 ms)Region opened successfully at 1733221368750 (+1 ms) 2024-12-03T10:22:48,757 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733221368659 2024-12-03T10:22:48,769 DEBUG [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T10:22:48,770 INFO [RS_OPEN_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T10:22:48,771 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:22:48,773 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a78f5601770a,38085,1733221335758, state=OPEN 2024-12-03T10:22:48,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T10:22:48,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T10:22:48,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T10:22:48,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T10:22:48,811 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a78f5601770a,38085,1733221335758 2024-12-03T10:22:48,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T10:22:48,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a78f5601770a,38085,1733221335758 in 323 msec 2024-12-03T10:22:48,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T10:22:48,824 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8400 sec 2024-12-03T10:22:48,825 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T10:22:48,825 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T10:22:48,843 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T10:22:48,844 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a78f5601770a,38085,1733221335758, seqNum=-1] 2024-12-03T10:22:48,863 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T10:22:48,866 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41361, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T10:22:48,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.8770 sec 2024-12-03T10:22:48,885 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733221368885, completionTime=-1 2024-12-03T10:22:48,889 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-03T10:22:48,889 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T10:22:48,915 INFO [master/a78f5601770a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-03T10:22:48,916 INFO [master/a78f5601770a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733221428916 2024-12-03T10:22:48,916 INFO [master/a78f5601770a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733221488916 2024-12-03T10:22:48,916 INFO [master/a78f5601770a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-12-03T10:22:48,918 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a78f5601770a,36731,1733221334944-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:48,918 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a78f5601770a,36731,1733221334944-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:48,918 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a78f5601770a,36731,1733221334944-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:48,920 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a78f5601770a:36731, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:48,920 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:48,921 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:48,926 DEBUG [master/a78f5601770a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T10:22:48,950 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.034sec 2024-12-03T10:22:48,952 INFO [master/a78f5601770a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T10:22:48,953 INFO [master/a78f5601770a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T10:22:48,954 INFO [master/a78f5601770a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T10:22:48,954 INFO [master/a78f5601770a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T10:22:48,955 INFO [master/a78f5601770a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T10:22:48,955 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a78f5601770a,36731,1733221334944-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T10:22:48,956 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a78f5601770a,36731,1733221334944-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T10:22:48,964 DEBUG [master/a78f5601770a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T10:22:48,965 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T10:22:48,965 INFO [master/a78f5601770a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a78f5601770a,36731,1733221334944-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T10:22:48,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc40f82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T10:22:48,998 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T10:22:48,998 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T10:22:49,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a78f5601770a,36731,-1 for getting cluster id 2024-12-03T10:22:49,004 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T10:22:49,012 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d730235-4285-4bc0-a468-a95c428290a0' 2024-12-03T10:22:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T10:22:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d730235-4285-4bc0-a468-a95c428290a0" 2024-12-03T10:22:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b758490, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T10:22:49,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a78f5601770a,36731,-1] 2024-12-03T10:22:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T10:22:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:22:49,037 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T10:22:49,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5413b239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T10:22:49,041 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T10:22:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a78f5601770a,38085,1733221335758, seqNum=-1] 2024-12-03T10:22:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T10:22:49,053 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T10:22:49,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a78f5601770a,36731,1733221334944 2024-12-03T10:22:49,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a2167c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T10:22:49,077 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a78f5601770a,36731,-1 for getting cluster id 2024-12-03T10:22:49,078 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T10:22:49,079 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d730235-4285-4bc0-a468-a95c428290a0' 2024-12-03T10:22:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T10:22:49,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d730235-4285-4bc0-a468-a95c428290a0" 2024-12-03T10:22:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd4d5f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T10:22:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a78f5601770a,36731,-1] 2024-12-03T10:22:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T10:22:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:22:49,083 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T10:22:49,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467ca67d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T10:22:49,095 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=9007 2024-12-03T10:22:49,100 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T10:22:49,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a78f5601770a,36731,1733221334944 2024-12-03T10:22:49,106 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ef18a0 2024-12-03T10:22:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T10:22:49,110 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T10:22:49,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:22:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T10:22:49,131 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:22:49,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-12-03T10:22:49,133 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:49,135 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:22:49,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T10:22:49,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741835_1011 (size=428) 2024-12-03T10:22:49,556 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff3b9ef17c9ec57d12f61a36f071de8d, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:22:49,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741836_1012 (size=79) 2024-12-03T10:22:49,966 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:22:49,966 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing ff3b9ef17c9ec57d12f61a36f071de8d, disabling compactions & flushes 2024-12-03T10:22:49,966 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:22:49,966 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:22:49,967 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. after waiting 0 ms 2024-12-03T10:22:49,967 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:22:49,967 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:22:49,967 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff3b9ef17c9ec57d12f61a36f071de8d: Waiting for close lock at 1733221369966Disabling compacts and flushes for region at 1733221369966Disabling writes for close at 1733221369967 (+1 ms)Writing region close event to WAL at 1733221369967Closed at 1733221369967 2024-12-03T10:22:49,971 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:22:49,977 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733221369971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221369971"}]},"ts":"1733221369971"} 2024-12-03T10:22:49,982 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:22:49,984 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:22:49,987 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221369984"}]},"ts":"1733221369984"} 2024-12-03T10:22:49,991 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-12-03T10:22:49,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=ff3b9ef17c9ec57d12f61a36f071de8d, ASSIGN}] 2024-12-03T10:22:49,995 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=ff3b9ef17c9ec57d12f61a36f071de8d, ASSIGN 2024-12-03T10:22:49,997 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=ff3b9ef17c9ec57d12f61a36f071de8d, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:22:50,151 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff3b9ef17c9ec57d12f61a36f071de8d, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:22:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T10:22:50,156 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=ff3b9ef17c9ec57d12f61a36f071de8d, ASSIGN because future has completed 2024-12-03T10:22:50,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff3b9ef17c9ec57d12f61a36f071de8d, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:22:50,318 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:22:50,318 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ff3b9ef17c9ec57d12f61a36f071de8d, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:22:50,319 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,319 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:22:50,319 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,319 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,322 INFO [StoreOpener-ff3b9ef17c9ec57d12f61a36f071de8d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,326 INFO [StoreOpener-ff3b9ef17c9ec57d12f61a36f071de8d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff3b9ef17c9ec57d12f61a36f071de8d columnFamilyName testFamily 2024-12-03T10:22:50,326 DEBUG [StoreOpener-ff3b9ef17c9ec57d12f61a36f071de8d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:50,327 INFO [StoreOpener-ff3b9ef17c9ec57d12f61a36f071de8d-1 {}] regionserver.HStore(327): Store=ff3b9ef17c9ec57d12f61a36f071de8d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:22:50,328 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,329 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,330 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,331 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,331 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,334 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,338 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:22:50,339 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ff3b9ef17c9ec57d12f61a36f071de8d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75046523, jitterRate=0.1182803362607956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:22:50,340 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:22:50,341 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ff3b9ef17c9ec57d12f61a36f071de8d: Running coprocessor pre-open hook at 1733221370319Writing region info on filesystem at 1733221370319Initializing all the Stores at 1733221370322 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221370322Cleaning up temporary data from old regions at 1733221370331 (+9 ms)Running coprocessor post-open hooks at 1733221370340 (+9 ms)Region opened successfully at 1733221370340 2024-12-03T10:22:50,343 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d., pid=6, masterSystemTime=1733221370311 2024-12-03T10:22:50,347 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:22:50,347 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:22:50,348 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ff3b9ef17c9ec57d12f61a36f071de8d, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:22:50,351 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff3b9ef17c9ec57d12f61a36f071de8d, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:22:50,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T10:22:50,357 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ff3b9ef17c9ec57d12f61a36f071de8d, server=a78f5601770a,38085,1733221335758 in 196 msec 2024-12-03T10:22:50,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T10:22:50,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=ff3b9ef17c9ec57d12f61a36f071de8d, ASSIGN in 364 msec 2024-12-03T10:22:50,362 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:22:50,363 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221370363"}]},"ts":"1733221370363"} 2024-12-03T10:22:50,366 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-12-03T10:22:50,368 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:22:50,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2460 sec 2024-12-03T10:22:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T10:22:52,170 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-12-03T10:22:52,170 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-12-03T10:22:52,173 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:22:52,179 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-12-03T10:22:52,180 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:22:52,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-12-03T10:22:52,183 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T10:22:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a78f5601770a,38085,1733221335758, seqNum=-1] 2024-12-03T10:22:52,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T10:22:52,188 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T10:22:52,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:22:52,209 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:22:52,213 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-12-03T10:22:52,217 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T10:22:52,222 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T10:22:52,223 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T10:22:52,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6255731883666046968 2024-12-03T10:22:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35336 deadline: 1733221375223, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T10:22:52,249 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T10:22:52,251 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T10:22:52,252 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:22:52,253 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:22:52,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6255731883666046966 2024-12-03T10:22:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35336 deadline: 1733221375253, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T10:22:52,258 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:22:52,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6255731883666046965 2024-12-03T10:22:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35336 deadline: 1733221375257, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T10:22:52,262 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:22:52,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6255731883666046964 2024-12-03T10:22:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35336 deadline: 1733221375261, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T10:22:52,266 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:22:52,276 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RS_OPEN_REGION-regionserver/a78f5601770a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_515815416_22 at /127.0.0.1:44910 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=198 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=8992 (was 9007) 2024-12-03T10:22:52,283 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=198, ProcessCount=11, AvailableMemoryMB=8992 2024-12-03T10:22:52,286 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:22:52,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-12-03T10:22:52,291 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:22:52,291 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:52,291 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-12-03T10:22:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T10:22:52,293 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:22:52,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741837_1013 (size=414) 2024-12-03T10:22:52,710 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6f6d5ad9ad336e2b65dfd8227b7adcf7, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:22:52,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741838_1014 (size=65) 2024-12-03T10:22:53,125 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:22:53,126 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 6f6d5ad9ad336e2b65dfd8227b7adcf7, disabling compactions & flushes 2024-12-03T10:22:53,126 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:22:53,126 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:22:53,127 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. after waiting 0 ms 2024-12-03T10:22:53,127 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:22:53,127 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:22:53,127 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6f6d5ad9ad336e2b65dfd8227b7adcf7: Waiting for close lock at 1733221373126Disabling compacts and flushes for region at 1733221373126Disabling writes for close at 1733221373127 (+1 ms)Writing region close event to WAL at 1733221373127Closed at 1733221373127 2024-12-03T10:22:53,132 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:22:53,133 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1733221373133"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221373133"}]},"ts":"1733221373133"} 2024-12-03T10:22:53,138 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:22:53,140 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:22:53,141 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221373140"}]},"ts":"1733221373140"} 2024-12-03T10:22:53,143 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T10:22:53,144 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=6f6d5ad9ad336e2b65dfd8227b7adcf7, ASSIGN}] 2024-12-03T10:22:53,146 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=6f6d5ad9ad336e2b65dfd8227b7adcf7, ASSIGN 2024-12-03T10:22:53,147 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=6f6d5ad9ad336e2b65dfd8227b7adcf7, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:22:53,298 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=6f6d5ad9ad336e2b65dfd8227b7adcf7, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:22:53,305 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=6f6d5ad9ad336e2b65dfd8227b7adcf7, ASSIGN because future has completed 2024-12-03T10:22:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T10:22:53,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f6d5ad9ad336e2b65dfd8227b7adcf7, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:22:53,475 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:22:53,475 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 6f6d5ad9ad336e2b65dfd8227b7adcf7, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:22:53,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:22:53,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,476 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,478 INFO [StoreOpener-6f6d5ad9ad336e2b65dfd8227b7adcf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,480 INFO [StoreOpener-6f6d5ad9ad336e2b65dfd8227b7adcf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6f6d5ad9ad336e2b65dfd8227b7adcf7 columnFamilyName testFamily 2024-12-03T10:22:53,480 DEBUG [StoreOpener-6f6d5ad9ad336e2b65dfd8227b7adcf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:22:53,481 INFO [StoreOpener-6f6d5ad9ad336e2b65dfd8227b7adcf7-1 {}] regionserver.HStore(327): Store=6f6d5ad9ad336e2b65dfd8227b7adcf7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:22:53,481 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,483 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,484 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,487 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,491 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:22:53,492 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 6f6d5ad9ad336e2b65dfd8227b7adcf7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71313182, jitterRate=0.06264922022819519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:22:53,492 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:22:53,493 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 6f6d5ad9ad336e2b65dfd8227b7adcf7: Running coprocessor pre-open hook at 1733221373476Writing region info on filesystem at 1733221373476Initializing all the Stores at 1733221373478 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221373478Cleaning up temporary data from old regions at 1733221373485 (+7 ms)Running coprocessor post-open hooks at 1733221373492 (+7 ms)Region opened successfully at 1733221373493 (+1 ms) 2024-12-03T10:22:53,495 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., pid=9, masterSystemTime=1733221373461 2024-12-03T10:22:53,498 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:22:53,498 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:22:53,499 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=6f6d5ad9ad336e2b65dfd8227b7adcf7, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:22:53,502 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6f6d5ad9ad336e2b65dfd8227b7adcf7, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:22:53,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-03T10:22:53,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 6f6d5ad9ad336e2b65dfd8227b7adcf7, server=a78f5601770a,38085,1733221335758 in 198 msec 2024-12-03T10:22:53,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T10:22:53,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=6f6d5ad9ad336e2b65dfd8227b7adcf7, ASSIGN in 364 msec 2024-12-03T10:22:53,513 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:22:53,513 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221373513"}]},"ts":"1733221373513"} 2024-12-03T10:22:53,516 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T10:22:53,518 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:22:53,521 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2320 sec 2024-12-03T10:22:54,256 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T10:22:54,280 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-12-03T10:22:54,281 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T10:22:54,282 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-12-03T10:22:55,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T10:22:55,305 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T10:22:55,308 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T10:22:55,308 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T10:22:55,308 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T10:22:55,308 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T10:22:55,310 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T10:22:55,310 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T10:22:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-03T10:22:55,326 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-12-03T10:22:55,326 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T10:22:55,326 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:22:55,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T10:22:55,331 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:22:55,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-12-03T10:22:55,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:22:55,346 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:22:55,346 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T10:22:55,349 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T10:22:55,351 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T10:23:01,365 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., hostname=a78f5601770a,38085,1733221335758, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., hostname=a78f5601770a,38085,1733221335758, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a78f5601770a:38085 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-12-03T10:23:01,366 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., hostname=a78f5601770a,38085,1733221335758, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-12-03T10:23:01,366 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., hostname=a78f5601770a,38085,1733221335758, seqNum=2 from cache 2024-12-03T10:23:01,374 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=182 (was 198), ProcessCount=11 (was 11), AvailableMemoryMB=8983 (was 8992) 2024-12-03T10:23:01,382 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=182, ProcessCount=11, AvailableMemoryMB=8983 2024-12-03T10:23:01,385 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:23:01,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-12-03T10:23:01,389 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:23:01,389 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:01,389 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-12-03T10:23:01,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:23:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T10:23:01,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741839_1015 (size=412) 2024-12-03T10:23:01,809 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c2af42950dba1c869d770f12c3be50ef, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:23:01,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741840_1016 (size=63) 2024-12-03T10:23:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-12-03T10:23:02,228 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:02,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing c2af42950dba1c869d770f12c3be50ef, disabling compactions & flushes 2024-12-03T10:23:02,229 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:02,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:02,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. after waiting 0 ms 2024-12-03T10:23:02,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:02,229 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:02,229 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for c2af42950dba1c869d770f12c3be50ef: Waiting for close lock at 1733221382229Disabling compacts and flushes for region at 1733221382229Disabling writes for close at 1733221382229Writing region close event to WAL at 1733221382229Closed at 1733221382229 2024-12-03T10:23:02,232 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:23:02,233 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1733221382233"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221382233"}]},"ts":"1733221382233"} 2024-12-03T10:23:02,238 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:23:02,240 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:23:02,240 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221382240"}]},"ts":"1733221382240"} 2024-12-03T10:23:02,244 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T10:23:02,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c2af42950dba1c869d770f12c3be50ef, ASSIGN}] 2024-12-03T10:23:02,246 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c2af42950dba1c869d770f12c3be50ef, ASSIGN 2024-12-03T10:23:02,248 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c2af42950dba1c869d770f12c3be50ef, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:23:02,399 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c2af42950dba1c869d770f12c3be50ef, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:02,405 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c2af42950dba1c869d770f12c3be50ef, ASSIGN because future has completed 2024-12-03T10:23:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T10:23:02,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2af42950dba1c869d770f12c3be50ef, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:23:02,573 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:02,573 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => c2af42950dba1c869d770f12c3be50ef, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:23:02,574 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,574 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:02,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,575 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,577 INFO [StoreOpener-c2af42950dba1c869d770f12c3be50ef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,579 INFO [StoreOpener-c2af42950dba1c869d770f12c3be50ef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2af42950dba1c869d770f12c3be50ef columnFamilyName testFamily 2024-12-03T10:23:02,579 DEBUG [StoreOpener-c2af42950dba1c869d770f12c3be50ef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:02,580 INFO [StoreOpener-c2af42950dba1c869d770f12c3be50ef-1 {}] regionserver.HStore(327): Store=c2af42950dba1c869d770f12c3be50ef/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:23:02,580 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,582 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,582 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,582 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,585 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:23:02,589 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened c2af42950dba1c869d770f12c3be50ef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70015576, jitterRate=0.04331338405609131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:23:02,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:02,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for c2af42950dba1c869d770f12c3be50ef: Running coprocessor pre-open hook at 1733221382575Writing region info on filesystem at 1733221382575Initializing all the Stores at 1733221382577 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221382577Cleaning up temporary data from old regions at 1733221382582 (+5 ms)Running coprocessor post-open hooks at 1733221382589 (+7 ms)Region opened successfully at 1733221382589 2024-12-03T10:23:02,591 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef., pid=12, masterSystemTime=1733221382562 2024-12-03T10:23:02,594 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:02,594 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:02,595 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=c2af42950dba1c869d770f12c3be50ef, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:02,598 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2af42950dba1c869d770f12c3be50ef, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:23:02,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-03T10:23:02,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure c2af42950dba1c869d770f12c3be50ef, server=a78f5601770a,38085,1733221335758 in 194 msec 2024-12-03T10:23:02,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T10:23:02,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=c2af42950dba1c869d770f12c3be50ef, ASSIGN in 359 msec 2024-12-03T10:23:02,608 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:23:02,608 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221382608"}]},"ts":"1733221382608"} 2024-12-03T10:23:02,610 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T10:23:02,612 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:23:02,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2270 sec 2024-12-03T10:23:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-12-03T10:23:04,419 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-12-03T10:23:04,419 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T10:23:04,420 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:04,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T10:23:04,430 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:04,430 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-12-03T10:23:04,433 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:04,445 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:23:04,445 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T10:23:04,448 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T10:23:04,450 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T10:23:04,452 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T10:23:04,453 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T10:23:05,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T10:23:05,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T10:23:06,285 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-12-03T10:23:08,111 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T10:23:10,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38085 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046955' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T10:23:10,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38085 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:35336 deadline: 1733221396456, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046955' does not exist 2024-12-03T10:23:10,470 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a78f5601770a,38085,1733221335758 for closing scanner id = -6255731883666046955 for c2af42950dba1c869d770f12c3be50ef of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046955' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T10:23:10,472 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=154 (was 182), ProcessCount=11 (was 11), AvailableMemoryMB=8969 (was 8983) 2024-12-03T10:23:10,480 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=154, ProcessCount=11, AvailableMemoryMB=8967 2024-12-03T10:23:10,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:23:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-12-03T10:23:10,486 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:23:10,486 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:10,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-12-03T10:23:10,487 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:23:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T10:23:10,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741841_1017 (size=423) 2024-12-03T10:23:10,904 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2eae6edbd9830e5528c525bf3f1f2923, NAME => 'testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:23:10,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741842_1018 (size=74) 2024-12-03T10:23:10,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-12-03T10:23:11,328 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:11,328 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 2eae6edbd9830e5528c525bf3f1f2923, disabling compactions & flushes 2024-12-03T10:23:11,328 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:11,329 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:11,329 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. after waiting 0 ms 2024-12-03T10:23:11,329 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:11,329 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:11,329 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2eae6edbd9830e5528c525bf3f1f2923: Waiting for close lock at 1733221391328Disabling compacts and flushes for region at 1733221391328Disabling writes for close at 1733221391329 (+1 ms)Writing region close event to WAL at 1733221391329Closed at 1733221391329 2024-12-03T10:23:11,333 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:23:11,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733221391333"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221391333"}]},"ts":"1733221391333"} 2024-12-03T10:23:11,337 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:23:11,339 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:23:11,339 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221391339"}]},"ts":"1733221391339"} 2024-12-03T10:23:11,343 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-12-03T10:23:11,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2eae6edbd9830e5528c525bf3f1f2923, ASSIGN}] 2024-12-03T10:23:11,346 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2eae6edbd9830e5528c525bf3f1f2923, ASSIGN 2024-12-03T10:23:11,347 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2eae6edbd9830e5528c525bf3f1f2923, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:23:11,498 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=2eae6edbd9830e5528c525bf3f1f2923, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:11,504 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2eae6edbd9830e5528c525bf3f1f2923, ASSIGN because future has completed 2024-12-03T10:23:11,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2eae6edbd9830e5528c525bf3f1f2923, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:23:11,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T10:23:11,671 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:11,671 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 2eae6edbd9830e5528c525bf3f1f2923, NAME => 'testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:23:11,672 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,672 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:11,672 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,672 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,674 INFO [StoreOpener-2eae6edbd9830e5528c525bf3f1f2923-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,677 INFO [StoreOpener-2eae6edbd9830e5528c525bf3f1f2923-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2eae6edbd9830e5528c525bf3f1f2923 columnFamilyName testFamily 2024-12-03T10:23:11,678 DEBUG [StoreOpener-2eae6edbd9830e5528c525bf3f1f2923-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:11,678 INFO [StoreOpener-2eae6edbd9830e5528c525bf3f1f2923-1 {}] regionserver.HStore(327): Store=2eae6edbd9830e5528c525bf3f1f2923/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:23:11,679 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,680 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,681 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,681 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,681 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,684 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,687 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:23:11,688 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 2eae6edbd9830e5528c525bf3f1f2923; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72973505, jitterRate=0.08738996088504791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:23:11,688 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:11,689 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 2eae6edbd9830e5528c525bf3f1f2923: Running coprocessor pre-open hook at 1733221391672Writing region info on filesystem at 1733221391672Initializing all the Stores at 1733221391674 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221391674Cleaning up temporary data from old regions at 1733221391681 (+7 ms)Running coprocessor post-open hooks at 1733221391688 (+7 ms)Region opened successfully at 1733221391689 (+1 ms) 2024-12-03T10:23:11,690 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923., pid=15, masterSystemTime=1733221391661 2024-12-03T10:23:11,693 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:11,693 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:11,694 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=2eae6edbd9830e5528c525bf3f1f2923, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:11,697 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2eae6edbd9830e5528c525bf3f1f2923, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:23:11,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T10:23:11,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 2eae6edbd9830e5528c525bf3f1f2923, server=a78f5601770a,38085,1733221335758 in 193 msec 2024-12-03T10:23:11,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T10:23:11,705 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=2eae6edbd9830e5528c525bf3f1f2923, ASSIGN in 358 msec 2024-12-03T10:23:11,706 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:23:11,706 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221391706"}]},"ts":"1733221391706"} 2024-12-03T10:23:11,709 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-12-03T10:23:11,710 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:23:11,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2270 sec 2024-12-03T10:23:12,288 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-12-03T10:23:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-03T10:23:13,518 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-12-03T10:23:13,518 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-12-03T10:23:13,519 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:13,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-12-03T10:23:13,529 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:13,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-12-03T10:23:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:13,544 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:23:13,544 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-12-03T10:23:13,547 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-12-03T10:23:13,549 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-12-03T10:23:13,550 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-12-03T10:23:13,551 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6255731883666046951 2024-12-03T10:23:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35336 deadline: 1733221396549, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T10:23:13,554 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-12-03T10:23:13,556 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-12-03T10:23:13,557 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:23:13,558 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:23:13,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-6255731883666046949 2024-12-03T10:23:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35336 deadline: 1733221396558, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T10:23:13,563 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:23:13,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-6255731883666046948 2024-12-03T10:23:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35336 deadline: 1733221396562, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T10:23:13,567 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:23:13,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-6255731883666046947 2024-12-03T10:23:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:35336 deadline: 1733221396566, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-12-03T10:23:13,571 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-12-03T10:23:13,581 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226 (was 225) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_515815416_22 at /127.0.0.1:34924 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 154), ProcessCount=11 (was 11), AvailableMemoryMB=8959 (was 8967) 2024-12-03T10:23:13,589 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=226, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=11, AvailableMemoryMB=8959 2024-12-03T10:23:13,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:23:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-12-03T10:23:13,596 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:23:13,596 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:13,596 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-12-03T10:23:13,598 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:23:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T10:23:13,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741843_1019 (size=418) 2024-12-03T10:23:14,015 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5cefb5eb6c037aa04998861c3c1edad3, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:23:14,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741844_1020 (size=69) 2024-12-03T10:23:14,026 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:14,027 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 5cefb5eb6c037aa04998861c3c1edad3, disabling compactions & flushes 2024-12-03T10:23:14,027 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:14,027 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:14,027 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. after waiting 0 ms 2024-12-03T10:23:14,027 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:14,027 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:14,027 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5cefb5eb6c037aa04998861c3c1edad3: Waiting for close lock at 1733221394026Disabling compacts and flushes for region at 1733221394026Disabling writes for close at 1733221394027 (+1 ms)Writing region close event to WAL at 1733221394027Closed at 1733221394027 2024-12-03T10:23:14,030 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:23:14,030 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1733221394030"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221394030"}]},"ts":"1733221394030"} 2024-12-03T10:23:14,033 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:23:14,034 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:23:14,034 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221394034"}]},"ts":"1733221394034"} 2024-12-03T10:23:14,037 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-12-03T10:23:14,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5cefb5eb6c037aa04998861c3c1edad3, ASSIGN}] 2024-12-03T10:23:14,038 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5cefb5eb6c037aa04998861c3c1edad3, ASSIGN 2024-12-03T10:23:14,039 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5cefb5eb6c037aa04998861c3c1edad3, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:23:14,089 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T10:23:14,191 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5cefb5eb6c037aa04998861c3c1edad3, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:14,200 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5cefb5eb6c037aa04998861c3c1edad3, ASSIGN because future has completed 2024-12-03T10:23:14,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cefb5eb6c037aa04998861c3c1edad3, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:23:14,367 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:14,367 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 5cefb5eb6c037aa04998861c3c1edad3, NAME => 'testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:23:14,367 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,367 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:14,368 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,368 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,369 INFO [StoreOpener-5cefb5eb6c037aa04998861c3c1edad3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,371 INFO [StoreOpener-5cefb5eb6c037aa04998861c3c1edad3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5cefb5eb6c037aa04998861c3c1edad3 columnFamilyName testFamily 2024-12-03T10:23:14,371 DEBUG [StoreOpener-5cefb5eb6c037aa04998861c3c1edad3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:14,372 INFO [StoreOpener-5cefb5eb6c037aa04998861c3c1edad3-1 {}] regionserver.HStore(327): Store=5cefb5eb6c037aa04998861c3c1edad3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:23:14,372 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,373 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,374 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,375 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,375 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,377 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,380 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:23:14,380 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 5cefb5eb6c037aa04998861c3c1edad3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67351004, jitterRate=0.0036081671714782715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:23:14,380 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:14,381 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 5cefb5eb6c037aa04998861c3c1edad3: Running coprocessor pre-open hook at 1733221394368Writing region info on filesystem at 1733221394368Initializing all the Stores at 1733221394369 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221394369Cleaning up temporary data from old regions at 1733221394375 (+6 ms)Running coprocessor post-open hooks at 1733221394380 (+5 ms)Region opened successfully at 1733221394381 (+1 ms) 2024-12-03T10:23:14,382 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., pid=18, masterSystemTime=1733221394356 2024-12-03T10:23:14,385 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:14,385 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:14,386 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=5cefb5eb6c037aa04998861c3c1edad3, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:14,389 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5cefb5eb6c037aa04998861c3c1edad3, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:23:14,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-03T10:23:14,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 5cefb5eb6c037aa04998861c3c1edad3, server=a78f5601770a,38085,1733221335758 in 189 msec 2024-12-03T10:23:14,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T10:23:14,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=5cefb5eb6c037aa04998861c3c1edad3, ASSIGN in 356 msec 2024-12-03T10:23:14,397 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:23:14,397 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221394397"}]},"ts":"1733221394397"} 2024-12-03T10:23:14,400 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-12-03T10:23:14,401 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:23:14,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 809 msec 2024-12-03T10:23:14,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-12-03T10:23:14,617 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-12-03T10:23:14,618 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-12-03T10:23:14,618 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:14,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-12-03T10:23:14,627 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:14,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-12-03T10:23:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:14,642 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:23:14,642 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-12-03T10:23:14,646 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T10:23:14,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T10:23:15,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T10:23:15,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-12-03T10:23:15,309 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T10:23:15,309 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-12-03T10:23:15,654 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a78f5601770a:38085 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-12-03T10:23:15,655 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1008ms, rpcTimeout=1000ms 2024-12-03T10:23:15,655 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 from cache 2024-12-03T10:23:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-12-03T10:23:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:16,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T10:23:17,176 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T10:23:17,654 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a78f5601770a:38085 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=975ms 2024-12-03T10:23:17,654 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=983ms, rpcTimeout=975ms 2024-12-03T10:23:17,655 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 from cache 2024-12-03T10:23:17,663 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223 (was 226), OpenFileDescriptor=449 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=142 (was 142), ProcessCount=11 (was 11), AvailableMemoryMB=8950 (was 8959) 2024-12-03T10:23:17,671 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=142, ProcessCount=11, AvailableMemoryMB=8950 2024-12-03T10:23:17,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:23:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-12-03T10:23:17,678 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:23:17,678 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:17,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-12-03T10:23:17,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T10:23:17,680 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:23:17,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741845_1021 (size=416) 2024-12-03T10:23:18,092 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df84c9bc6c4e06a825ca56632528287e, NAME => 'testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:23:18,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741846_1022 (size=67) 2024-12-03T10:23:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-12-03T10:23:18,291 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-12-03T10:23:18,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:18,507 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing df84c9bc6c4e06a825ca56632528287e, disabling compactions & flushes 2024-12-03T10:23:18,508 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:18,508 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:18,508 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. after waiting 0 ms 2024-12-03T10:23:18,508 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:18,508 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:18,509 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for df84c9bc6c4e06a825ca56632528287e: Waiting for close lock at 1733221398507Disabling compacts and flushes for region at 1733221398507Disabling writes for close at 1733221398508 (+1 ms)Writing region close event to WAL at 1733221398508Closed at 1733221398508 2024-12-03T10:23:18,512 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:23:18,513 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1733221398512"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221398512"}]},"ts":"1733221398512"} 2024-12-03T10:23:18,518 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:23:18,520 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:23:18,520 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221398520"}]},"ts":"1733221398520"} 2024-12-03T10:23:18,523 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-12-03T10:23:18,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=df84c9bc6c4e06a825ca56632528287e, ASSIGN}] 2024-12-03T10:23:18,525 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=df84c9bc6c4e06a825ca56632528287e, ASSIGN 2024-12-03T10:23:18,526 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=df84c9bc6c4e06a825ca56632528287e, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:23:18,677 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=df84c9bc6c4e06a825ca56632528287e, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:18,683 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=df84c9bc6c4e06a825ca56632528287e, ASSIGN because future has completed 2024-12-03T10:23:18,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure df84c9bc6c4e06a825ca56632528287e, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:23:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T10:23:18,850 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:18,851 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => df84c9bc6c4e06a825ca56632528287e, NAME => 'testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:23:18,851 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,852 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:18,852 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,852 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,854 INFO [StoreOpener-df84c9bc6c4e06a825ca56632528287e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,856 INFO [StoreOpener-df84c9bc6c4e06a825ca56632528287e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df84c9bc6c4e06a825ca56632528287e columnFamilyName testFamily 2024-12-03T10:23:18,856 DEBUG [StoreOpener-df84c9bc6c4e06a825ca56632528287e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:18,857 INFO [StoreOpener-df84c9bc6c4e06a825ca56632528287e-1 {}] regionserver.HStore(327): Store=df84c9bc6c4e06a825ca56632528287e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:23:18,857 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,858 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,859 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,859 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,859 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,862 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,865 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:23:18,865 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened df84c9bc6c4e06a825ca56632528287e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59170819, jitterRate=-0.11828608810901642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:23:18,865 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:18,866 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for df84c9bc6c4e06a825ca56632528287e: Running coprocessor pre-open hook at 1733221398852Writing region info on filesystem at 1733221398852Initializing all the Stores at 1733221398854 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221398854Cleaning up temporary data from old regions at 1733221398859 (+5 ms)Running coprocessor post-open hooks at 1733221398865 (+6 ms)Region opened successfully at 1733221398866 (+1 ms) 2024-12-03T10:23:18,867 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e., pid=21, masterSystemTime=1733221398839 2024-12-03T10:23:18,870 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:18,870 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:18,870 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=df84c9bc6c4e06a825ca56632528287e, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:18,873 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure df84c9bc6c4e06a825ca56632528287e, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:23:18,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-03T10:23:18,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure df84c9bc6c4e06a825ca56632528287e, server=a78f5601770a,38085,1733221335758 in 190 msec 2024-12-03T10:23:18,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-03T10:23:18,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=df84c9bc6c4e06a825ca56632528287e, ASSIGN in 354 msec 2024-12-03T10:23:18,881 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:23:18,881 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221398881"}]},"ts":"1733221398881"} 2024-12-03T10:23:18,884 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-12-03T10:23:18,885 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:23:18,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2110 sec 2024-12-03T10:23:20,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-03T10:23:20,719 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-12-03T10:23:20,719 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-12-03T10:23:20,720 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:20,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-12-03T10:23:20,730 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:20,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-12-03T10:23:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:20,745 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:23:20,745 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-12-03T10:23:20,748 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T10:23:20,751 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T10:23:20,752 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T10:23:20,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T10:23:23,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38085 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046936' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T10:23:23,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38085 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:35336 deadline: 1733221404756, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046936' does not exist 2024-12-03T10:23:23,763 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a78f5601770a,38085,1733221335758 for closing scanner id = -6255731883666046936 for df84c9bc6c4e06a825ca56632528287e of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046936' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T10:23:23,770 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 223), OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=120 (was 142), ProcessCount=11 (was 11), AvailableMemoryMB=8953 (was 8950) - AvailableMemoryMB LEAK? - 2024-12-03T10:23:23,781 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=120, ProcessCount=11, AvailableMemoryMB=8953 2024-12-03T10:23:23,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:23:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-12-03T10:23:23,789 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:23:23,789 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:23,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-12-03T10:23:23,791 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:23:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T10:23:23,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741847_1023 (size=411) 2024-12-03T10:23:24,204 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 34d30e8292e4b552cdf4f07ecdd16178, NAME => 'testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:23:24,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741848_1024 (size=62) 2024-12-03T10:23:24,254 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-12-03T10:23:24,355 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T10:23:24,382 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-12-03T10:23:24,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:24,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 34d30e8292e4b552cdf4f07ecdd16178, disabling compactions & flushes 2024-12-03T10:23:24,620 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:24,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:24,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. after waiting 0 ms 2024-12-03T10:23:24,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:24,620 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:24,620 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 34d30e8292e4b552cdf4f07ecdd16178: Waiting for close lock at 1733221404620Disabling compacts and flushes for region at 1733221404620Disabling writes for close at 1733221404620Writing region close event to WAL at 1733221404620Closed at 1733221404620 2024-12-03T10:23:24,622 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:23:24,622 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1733221404622"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221404622"}]},"ts":"1733221404622"} 2024-12-03T10:23:24,625 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:23:24,627 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:23:24,627 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221404627"}]},"ts":"1733221404627"} 2024-12-03T10:23:24,630 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T10:23:24,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=34d30e8292e4b552cdf4f07ecdd16178, ASSIGN}] 2024-12-03T10:23:24,632 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=34d30e8292e4b552cdf4f07ecdd16178, ASSIGN 2024-12-03T10:23:24,633 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=34d30e8292e4b552cdf4f07ecdd16178, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:23:24,784 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=34d30e8292e4b552cdf4f07ecdd16178, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:24,790 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=34d30e8292e4b552cdf4f07ecdd16178, ASSIGN because future has completed 2024-12-03T10:23:24,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34d30e8292e4b552cdf4f07ecdd16178, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:23:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T10:23:24,955 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:24,955 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 34d30e8292e4b552cdf4f07ecdd16178, NAME => 'testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:23:24,956 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,956 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:24,956 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,956 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,959 INFO [StoreOpener-34d30e8292e4b552cdf4f07ecdd16178-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,962 INFO [StoreOpener-34d30e8292e4b552cdf4f07ecdd16178-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34d30e8292e4b552cdf4f07ecdd16178 columnFamilyName testFamily 2024-12-03T10:23:24,962 DEBUG [StoreOpener-34d30e8292e4b552cdf4f07ecdd16178-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:24,963 INFO [StoreOpener-34d30e8292e4b552cdf4f07ecdd16178-1 {}] regionserver.HStore(327): Store=34d30e8292e4b552cdf4f07ecdd16178/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:23:24,963 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,964 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,965 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,965 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,965 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,968 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,971 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:23:24,971 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 34d30e8292e4b552cdf4f07ecdd16178; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72794080, jitterRate=0.0847163200378418}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:23:24,972 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:24,972 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 34d30e8292e4b552cdf4f07ecdd16178: Running coprocessor pre-open hook at 1733221404956Writing region info on filesystem at 1733221404956Initializing all the Stores at 1733221404958 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221404958Cleaning up temporary data from old regions at 1733221404965 (+7 ms)Running coprocessor post-open hooks at 1733221404972 (+7 ms)Region opened successfully at 1733221404972 2024-12-03T10:23:24,973 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178., pid=24, masterSystemTime=1733221404946 2024-12-03T10:23:24,976 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:24,976 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:24,977 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=34d30e8292e4b552cdf4f07ecdd16178, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:24,979 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34d30e8292e4b552cdf4f07ecdd16178, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:23:24,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-03T10:23:24,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 34d30e8292e4b552cdf4f07ecdd16178, server=a78f5601770a,38085,1733221335758 in 190 msec 2024-12-03T10:23:24,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-03T10:23:24,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=34d30e8292e4b552cdf4f07ecdd16178, ASSIGN in 353 msec 2024-12-03T10:23:24,987 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:23:24,987 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221404987"}]},"ts":"1733221404987"} 2024-12-03T10:23:24,990 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T10:23:24,991 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:23:24,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2070 sec 2024-12-03T10:23:25,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T10:23:25,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T10:23:25,308 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T10:23:25,308 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T10:23:26,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-03T10:23:26,816 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-12-03T10:23:26,816 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T10:23:26,816 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:26,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T10:23:26,824 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:26,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-12-03T10:23:26,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:26,838 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:23:26,838 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T10:23:26,840 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T10:23:26,842 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T10:23:26,843 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T10:23:26,844 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-12-03T10:23:28,970 INFO [master/a78f5601770a:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T10:23:28,970 INFO [master/a78f5601770a:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T10:23:29,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38085 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046932' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T10:23:29,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38085 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:35336 deadline: 1733221410850, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046932' does not exist 2024-12-03T10:23:29,852 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a78f5601770a,38085,1733221335758 for closing scanner id = -6255731883666046932 for 34d30e8292e4b552cdf4f07ecdd16178 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046932' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T10:23:29,858 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=110 (was 120), ProcessCount=11 (was 11), AvailableMemoryMB=8949 (was 8953) 2024-12-03T10:23:29,867 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=110, ProcessCount=11, AvailableMemoryMB=8949 2024-12-03T10:23:29,869 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:23:29,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T10:23:29,872 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:23:29,872 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:29,872 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-12-03T10:23:29,873 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:23:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T10:23:29,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741849_1025 (size=419) 2024-12-03T10:23:30,286 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e38f183a25993c2b5fc891f9fc88f9df, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:23:30,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741850_1026 (size=70) 2024-12-03T10:23:30,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-12-03T10:23:30,459 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T10:23:30,489 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-12-03T10:23:30,702 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:30,702 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing e38f183a25993c2b5fc891f9fc88f9df, disabling compactions & flushes 2024-12-03T10:23:30,702 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:30,703 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:30,703 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. after waiting 0 ms 2024-12-03T10:23:30,703 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:30,703 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:30,703 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for e38f183a25993c2b5fc891f9fc88f9df: Waiting for close lock at 1733221410702Disabling compacts and flushes for region at 1733221410702Disabling writes for close at 1733221410703 (+1 ms)Writing region close event to WAL at 1733221410703Closed at 1733221410703 2024-12-03T10:23:30,708 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:23:30,709 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1733221410708"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221410708"}]},"ts":"1733221410708"} 2024-12-03T10:23:30,713 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:23:30,715 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:23:30,715 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221410715"}]},"ts":"1733221410715"} 2024-12-03T10:23:30,717 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T10:23:30,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e38f183a25993c2b5fc891f9fc88f9df, ASSIGN}] 2024-12-03T10:23:30,719 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e38f183a25993c2b5fc891f9fc88f9df, ASSIGN 2024-12-03T10:23:30,720 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e38f183a25993c2b5fc891f9fc88f9df, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:23:30,871 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=e38f183a25993c2b5fc891f9fc88f9df, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:30,878 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e38f183a25993c2b5fc891f9fc88f9df, ASSIGN because future has completed 2024-12-03T10:23:30,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure e38f183a25993c2b5fc891f9fc88f9df, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:23:30,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T10:23:31,043 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:31,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => e38f183a25993c2b5fc891f9fc88f9df, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:23:31,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:31,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,044 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,047 INFO [StoreOpener-e38f183a25993c2b5fc891f9fc88f9df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,050 INFO [StoreOpener-e38f183a25993c2b5fc891f9fc88f9df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e38f183a25993c2b5fc891f9fc88f9df columnFamilyName testFamily 2024-12-03T10:23:31,050 DEBUG [StoreOpener-e38f183a25993c2b5fc891f9fc88f9df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:31,051 INFO [StoreOpener-e38f183a25993c2b5fc891f9fc88f9df-1 {}] regionserver.HStore(327): Store=e38f183a25993c2b5fc891f9fc88f9df/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:23:31,051 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,052 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,054 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,055 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,055 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,057 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,060 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:23:31,061 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened e38f183a25993c2b5fc891f9fc88f9df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72764872, jitterRate=0.0842810869216919}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:23:31,061 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:31,062 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for e38f183a25993c2b5fc891f9fc88f9df: Running coprocessor pre-open hook at 1733221411045Writing region info on filesystem at 1733221411045Initializing all the Stores at 1733221411046 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221411047 (+1 ms)Cleaning up temporary data from old regions at 1733221411055 (+8 ms)Running coprocessor post-open hooks at 1733221411061 (+6 ms)Region opened successfully at 1733221411062 (+1 ms) 2024-12-03T10:23:31,063 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., pid=27, masterSystemTime=1733221411031 2024-12-03T10:23:31,065 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:31,065 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:31,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=e38f183a25993c2b5fc891f9fc88f9df, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:31,068 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure e38f183a25993c2b5fc891f9fc88f9df, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:23:31,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-03T10:23:31,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure e38f183a25993c2b5fc891f9fc88f9df, server=a78f5601770a,38085,1733221335758 in 192 msec 2024-12-03T10:23:31,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-03T10:23:31,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=e38f183a25993c2b5fc891f9fc88f9df, ASSIGN in 354 msec 2024-12-03T10:23:31,075 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:23:31,076 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221411075"}]},"ts":"1733221411075"} 2024-12-03T10:23:31,078 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T10:23:31,079 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:23:31,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2100 sec 2024-12-03T10:23:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-03T10:23:32,909 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-12-03T10:23:32,909 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T10:23:32,909 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:32,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T10:23:32,919 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:32,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T10:23:32,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:32,935 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:23:32,935 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T10:23:32,938 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T10:23:32,940 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-12-03T10:23:35,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T10:23:35,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-12-03T10:23:36,548 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T10:23:36,572 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-12-03T10:23:38,945 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., hostname=a78f5601770a,38085,1733221335758, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., hostname=a78f5601770a,38085,1733221335758, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a78f5601770a:38085 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-12-03T10:23:38,945 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., hostname=a78f5601770a,38085,1733221335758, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6005ms, rpcTimeout=5999ms 2024-12-03T10:23:38,945 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., hostname=a78f5601770a,38085,1733221335758, seqNum=2 from cache 2024-12-03T10:23:38,961 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=93 (was 110), ProcessCount=11 (was 11), AvailableMemoryMB=8946 (was 8949) 2024-12-03T10:23:38,971 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=93, ProcessCount=11, AvailableMemoryMB=8945 2024-12-03T10:23:38,974 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:23:38,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-12-03T10:23:38,977 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:23:38,977 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:38,977 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-12-03T10:23:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T10:23:38,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:23:38,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741851_1027 (size=407) 2024-12-03T10:23:38,991 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 80b17e09311ca90d67a44c0da1802b88, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:23:38,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741852_1028 (size=58) 2024-12-03T10:23:38,999 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:39,000 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 80b17e09311ca90d67a44c0da1802b88, disabling compactions & flushes 2024-12-03T10:23:39,000 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:39,000 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:39,000 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. after waiting 0 ms 2024-12-03T10:23:39,000 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:39,000 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:39,000 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 80b17e09311ca90d67a44c0da1802b88: Waiting for close lock at 1733221419000Disabling compacts and flushes for region at 1733221419000Disabling writes for close at 1733221419000Writing region close event to WAL at 1733221419000Closed at 1733221419000 2024-12-03T10:23:39,001 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:23:39,001 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1733221419001"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221419001"}]},"ts":"1733221419001"} 2024-12-03T10:23:39,004 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:23:39,005 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:23:39,005 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221419005"}]},"ts":"1733221419005"} 2024-12-03T10:23:39,007 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-12-03T10:23:39,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=80b17e09311ca90d67a44c0da1802b88, ASSIGN}] 2024-12-03T10:23:39,009 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=80b17e09311ca90d67a44c0da1802b88, ASSIGN 2024-12-03T10:23:39,010 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=80b17e09311ca90d67a44c0da1802b88, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:23:39,161 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=80b17e09311ca90d67a44c0da1802b88, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:39,168 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=80b17e09311ca90d67a44c0da1802b88, ASSIGN because future has completed 2024-12-03T10:23:39,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80b17e09311ca90d67a44c0da1802b88, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:23:39,333 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:39,334 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 80b17e09311ca90d67a44c0da1802b88, NAME => 'hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:23:39,335 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,335 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:39,335 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,335 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,339 INFO [StoreOpener-80b17e09311ca90d67a44c0da1802b88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,342 INFO [StoreOpener-80b17e09311ca90d67a44c0da1802b88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 80b17e09311ca90d67a44c0da1802b88 columnFamilyName testFamily 2024-12-03T10:23:39,342 DEBUG [StoreOpener-80b17e09311ca90d67a44c0da1802b88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:39,343 INFO [StoreOpener-80b17e09311ca90d67a44c0da1802b88-1 {}] regionserver.HStore(327): Store=80b17e09311ca90d67a44c0da1802b88/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:23:39,343 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,345 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,346 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,347 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,347 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,350 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,353 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:23:39,354 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 80b17e09311ca90d67a44c0da1802b88; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73044150, jitterRate=0.08844265341758728}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:23:39,354 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:39,355 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 80b17e09311ca90d67a44c0da1802b88: Running coprocessor pre-open hook at 1733221419336Writing region info on filesystem at 1733221419336Initializing all the Stores at 1733221419338 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221419338Cleaning up temporary data from old regions at 1733221419347 (+9 ms)Running coprocessor post-open hooks at 1733221419354 (+7 ms)Region opened successfully at 1733221419355 (+1 ms) 2024-12-03T10:23:39,356 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88., pid=30, masterSystemTime=1733221419322 2024-12-03T10:23:39,358 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:39,358 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:39,359 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=80b17e09311ca90d67a44c0da1802b88, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:39,362 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 80b17e09311ca90d67a44c0da1802b88, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:23:39,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-03T10:23:39,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 80b17e09311ca90d67a44c0da1802b88, server=a78f5601770a,38085,1733221335758 in 195 msec 2024-12-03T10:23:39,368 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-12-03T10:23:39,369 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=80b17e09311ca90d67a44c0da1802b88, ASSIGN in 359 msec 2024-12-03T10:23:39,369 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:23:39,370 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221419370"}]},"ts":"1733221419370"} 2024-12-03T10:23:39,372 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-12-03T10:23:39,373 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:23:39,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 399 msec 2024-12-03T10:23:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-12-03T10:23:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-12-03T10:23:39,999 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-12-03T10:23:39,999 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-12-03T10:23:40,000 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:40,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-12-03T10:23:40,009 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:40,009 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-12-03T10:23:40,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:40,024 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:23:40,024 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-12-03T10:23:40,028 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-12-03T10:23:40,030 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-12-03T10:23:40,031 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-12-03T10:23:40,031 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-12-03T10:23:42,573 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-12-03T10:23:44,090 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T10:23:44,839 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T10:23:45,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T10:23:45,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-12-03T10:23:46,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38085 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046924' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-12-03T10:23:46,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38085 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:35336 deadline: 1733221432040, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046924' does not exist 2024-12-03T10:23:46,042 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to a78f5601770a,38085,1733221335758 for closing scanner id = -6255731883666046924 for 80b17e09311ca90d67a44c0da1802b88 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-6255731883666046924' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T10:23:46,049 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=86 (was 93), ProcessCount=11 (was 11), AvailableMemoryMB=8934 (was 8945) 2024-12-03T10:23:46,057 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=86, ProcessCount=11, AvailableMemoryMB=8934 2024-12-03T10:23:46,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T10:23:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-12-03T10:23:46,061 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T10:23:46,061 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:46,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-12-03T10:23:46,062 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T10:23:46,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T10:23:46,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741853_1029 (size=423) 2024-12-03T10:23:46,479 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f69189b7b7330efefa5b500ab3856ac3, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2 2024-12-03T10:23:46,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741854_1030 (size=74) 2024-12-03T10:23:46,492 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:46,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing f69189b7b7330efefa5b500ab3856ac3, disabling compactions & flushes 2024-12-03T10:23:46,493 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:46,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:46,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. after waiting 0 ms 2024-12-03T10:23:46,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:46,493 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:46,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f69189b7b7330efefa5b500ab3856ac3: Waiting for close lock at 1733221426492Disabling compacts and flushes for region at 1733221426492Disabling writes for close at 1733221426493 (+1 ms)Writing region close event to WAL at 1733221426493Closed at 1733221426493 2024-12-03T10:23:46,495 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T10:23:46,495 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1733221426495"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733221426495"}]},"ts":"1733221426495"} 2024-12-03T10:23:46,498 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T10:23:46,499 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T10:23:46,499 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221426499"}]},"ts":"1733221426499"} 2024-12-03T10:23:46,501 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-12-03T10:23:46,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=f69189b7b7330efefa5b500ab3856ac3, ASSIGN}] 2024-12-03T10:23:46,502 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=f69189b7b7330efefa5b500ab3856ac3, ASSIGN 2024-12-03T10:23:46,503 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=f69189b7b7330efefa5b500ab3856ac3, ASSIGN; state=OFFLINE, location=a78f5601770a,38085,1733221335758; forceNewPlan=false, retain=false 2024-12-03T10:23:46,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-12-03T10:23:46,655 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=f69189b7b7330efefa5b500ab3856ac3, regionState=OPENING, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:46,662 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=f69189b7b7330efefa5b500ab3856ac3, ASSIGN because future has completed 2024-12-03T10:23:46,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure f69189b7b7330efefa5b500ab3856ac3, server=a78f5601770a,38085,1733221335758}] 2024-12-03T10:23:46,826 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:46,827 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => f69189b7b7330efefa5b500ab3856ac3, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T10:23:46,827 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,827 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T10:23:46,828 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,828 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,830 INFO [StoreOpener-f69189b7b7330efefa5b500ab3856ac3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,833 INFO [StoreOpener-f69189b7b7330efefa5b500ab3856ac3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f69189b7b7330efefa5b500ab3856ac3 columnFamilyName testFamily 2024-12-03T10:23:46,833 DEBUG [StoreOpener-f69189b7b7330efefa5b500ab3856ac3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T10:23:46,834 INFO [StoreOpener-f69189b7b7330efefa5b500ab3856ac3-1 {}] regionserver.HStore(327): Store=f69189b7b7330efefa5b500ab3856ac3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T10:23:46,834 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,835 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,836 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,837 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,837 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,840 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,844 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T10:23:46,845 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened f69189b7b7330efefa5b500ab3856ac3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66652426, jitterRate=-0.0068014562129974365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T10:23:46,845 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:46,846 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for f69189b7b7330efefa5b500ab3856ac3: Running coprocessor pre-open hook at 1733221426828Writing region info on filesystem at 1733221426828Initializing all the Stores at 1733221426830 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733221426830Cleaning up temporary data from old regions at 1733221426837 (+7 ms)Running coprocessor post-open hooks at 1733221426845 (+8 ms)Region opened successfully at 1733221426846 (+1 ms) 2024-12-03T10:23:46,847 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., pid=33, masterSystemTime=1733221426818 2024-12-03T10:23:46,850 DEBUG [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:46,850 INFO [RS_OPEN_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:46,851 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=f69189b7b7330efefa5b500ab3856ac3, regionState=OPEN, openSeqNum=2, regionLocation=a78f5601770a,38085,1733221335758 2024-12-03T10:23:46,854 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure f69189b7b7330efefa5b500ab3856ac3, server=a78f5601770a,38085,1733221335758 because future has completed 2024-12-03T10:23:46,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-03T10:23:46,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure f69189b7b7330efefa5b500ab3856ac3, server=a78f5601770a,38085,1733221335758 in 194 msec 2024-12-03T10:23:46,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-03T10:23:46,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=f69189b7b7330efefa5b500ab3856ac3, ASSIGN in 357 msec 2024-12-03T10:23:46,862 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T10:23:46,862 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733221426862"}]},"ts":"1733221426862"} 2024-12-03T10:23:46,864 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-12-03T10:23:46,865 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T10:23:46,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 806 msec 2024-12-03T10:23:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36731 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-03T10:23:47,076 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-12-03T10:23:47,076 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-12-03T10:23:47,076 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:47,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-12-03T10:23:47,083 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T10:23:47,083 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-12-03T10:23:47,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:47,099 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-12-03T10:23:47,099 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-12-03T10:23:47,103 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-12-03T10:23:47,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T10:23:48,114 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a78f5601770a:38085 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1011ms, rpcTimeout=1000ms 2024-12-03T10:23:48,114 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1011ms, rpcTimeout=1000ms 2024-12-03T10:23:48,114 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 from cache 2024-12-03T10:23:48,577 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-12-03T10:23:48,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-12-03T10:23:48,933 DEBUG [master/a78f5601770a:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T10:23:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2] 2024-12-03T10:23:49,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38085 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-12-03T10:23:50,104 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=a78f5601770a:38085 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=972ms, rpcTimeout=971ms 2024-12-03T10:23:50,105 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=972ms, rpcTimeout=971ms 2024-12-03T10:23:50,105 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., hostname=a78f5601770a,38085,1733221335758, seqNum=2 from cache 2024-12-03T10:23:50,120 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=224 (was 222) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_515815416_22 at /127.0.0.1:40790 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=79 (was 86), ProcessCount=11 (was 11), AvailableMemoryMB=8932 (was 8934) 2024-12-03T10:23:50,121 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T10:23:50,121 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T10:23:50,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:23:50,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:23:50,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T10:23:50,128 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T10:23:50,128 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T10:23:50,128 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T10:23:50,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:23:50,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:23:50,128 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T10:23:50,128 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T10:23:50,129 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=331909808, stopped=false 2024-12-03T10:23:50,129 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a78f5601770a,36731,1733221334944 2024-12-03T10:23:50,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T10:23:50,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T10:23:50,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:23:50,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:23:50,185 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T10:23:50,187 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T10:23:50,187 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T10:23:50,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:23:50,187 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T10:23:50,187 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T10:23:50,188 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a78f5601770a,38085,1733221335758' ***** 2024-12-03T10:23:50,188 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T10:23:50,188 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T10:23:50,188 INFO [RS:0;a78f5601770a:38085 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T10:23:50,188 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T10:23:50,188 INFO [RS:0;a78f5601770a:38085 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T10:23:50,188 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:23:50,189 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for 5cefb5eb6c037aa04998861c3c1edad3 2024-12-03T10:23:50,189 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:50,189 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for 34d30e8292e4b552cdf4f07ecdd16178 2024-12-03T10:23:50,189 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:50,189 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:50,189 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff3b9ef17c9ec57d12f61a36f071de8d, disabling compactions & flushes 2024-12-03T10:23:50,189 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:50,189 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:23:50,189 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for 2eae6edbd9830e5528c525bf3f1f2923 2024-12-03T10:23:50,189 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:23:50,189 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for 6f6d5ad9ad336e2b65dfd8227b7adcf7 2024-12-03T10:23:50,189 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. after waiting 0 ms 2024-12-03T10:23:50,190 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:23:50,190 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:50,190 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(959): stopping server a78f5601770a,38085,1733221335758 2024-12-03T10:23:50,190 INFO [RS:0;a78f5601770a:38085 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T10:23:50,190 INFO [RS:0;a78f5601770a:38085 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a78f5601770a:38085. 2024-12-03T10:23:50,190 DEBUG [RS:0;a78f5601770a:38085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T10:23:50,190 DEBUG [RS:0;a78f5601770a:38085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:23:50,190 INFO [RS:0;a78f5601770a:38085 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T10:23:50,190 INFO [RS:0;a78f5601770a:38085 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T10:23:50,190 INFO [RS:0;a78f5601770a:38085 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T10:23:50,190 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T10:23:50,191 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-12-03T10:23:50,191 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T10:23:50,191 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T10:23:50,191 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T10:23:50,191 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T10:23:50,191 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T10:23:50,191 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 5cefb5eb6c037aa04998861c3c1edad3, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df, f69189b7b7330efefa5b500ab3856ac3, ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:23:50,193 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-12-03T10:23:50,193 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ff3b9ef17c9ec57d12f61a36f071de8d 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:50,231 INFO [regionserver/a78f5601770a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T10:23:50,232 INFO [regionserver/a78f5601770a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T10:23:50,240 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d/.tmp/testFamily/9ed1899a3bfd4937b68a516827610515 is 61, key is row-0/testFamily:testQualifier/1733221372197/Put/seqid=0 2024-12-03T10:23:50,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741855_1031 (size=5049) 2024-12-03T10:23:50,250 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/info/3e64c4e4cbde49aabc500540346288d2 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d./info:regioninfo/1733221370348/Put/seqid=0 2024-12-03T10:23:50,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741856_1032 (size=20564) 2024-12-03T10:23:50,392 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 5cefb5eb6c037aa04998861c3c1edad3, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df, f69189b7b7330efefa5b500ab3856ac3, ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:23:50,592 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 5cefb5eb6c037aa04998861c3c1edad3, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df, f69189b7b7330efefa5b500ab3856ac3, ff3b9ef17c9ec57d12f61a36f071de8d 2024-12-03T10:23:50,635 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-12-03T10:23:50,650 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d/.tmp/testFamily/9ed1899a3bfd4937b68a516827610515 2024-12-03T10:23:50,658 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/info/3e64c4e4cbde49aabc500540346288d2 2024-12-03T10:23:50,698 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d/.tmp/testFamily/9ed1899a3bfd4937b68a516827610515 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d/testFamily/9ed1899a3bfd4937b68a516827610515 2024-12-03T10:23:50,707 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d/testFamily/9ed1899a3bfd4937b68a516827610515, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:50,713 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ff3b9ef17c9ec57d12f61a36f071de8d in 519ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:50,713 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/ns/68f3b9b5894c4e84be931a0acc64c821 is 43, key is default/ns:d/1733221368870/Put/seqid=0 2024-12-03T10:23:50,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741857_1033 (size=5153) 2024-12-03T10:23:50,720 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextExceptionAsync/ff3b9ef17c9ec57d12f61a36f071de8d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:50,722 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:23:50,722 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff3b9ef17c9ec57d12f61a36f071de8d: Waiting for close lock at 1733221430189Running coprocessor pre-close hooks at 1733221430189Disabling compacts and flushes for region at 1733221430189Disabling writes for close at 1733221430190 (+1 ms)Obtaining lock to block concurrent updates at 1733221430193 (+3 ms)Preparing flush snapshotting stores in ff3b9ef17c9ec57d12f61a36f071de8d at 1733221430193Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221430199 (+6 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. at 1733221430201 (+2 ms)Flushing ff3b9ef17c9ec57d12f61a36f071de8d/testFamily: creating writer at 1733221430202 (+1 ms)Flushing ff3b9ef17c9ec57d12f61a36f071de8d/testFamily: appending metadata at 1733221430235 (+33 ms)Flushing ff3b9ef17c9ec57d12f61a36f071de8d/testFamily: closing flushed file at 1733221430237 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6717fe65: reopening flushed file at 1733221430696 (+459 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ff3b9ef17c9ec57d12f61a36f071de8d in 519ms, sequenceid=8, compaction requested=false at 1733221430713 (+17 ms)Writing region close event to WAL at 1733221430715 (+2 ms)Running coprocessor post-close hooks at 1733221430721 (+6 ms)Closed at 1733221430722 (+1 ms) 2024-12-03T10:23:50,723 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1733221369112.ff3b9ef17c9ec57d12f61a36f071de8d. 2024-12-03T10:23:50,723 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5cefb5eb6c037aa04998861c3c1edad3, disabling compactions & flushes 2024-12-03T10:23:50,723 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:50,723 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:50,723 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. after waiting 0 ms 2024-12-03T10:23:50,723 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:50,723 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5cefb5eb6c037aa04998861c3c1edad3 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:50,738 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3/.tmp/testFamily/3ae68a46c6fb4ca898368805b12faefe is 61, key is row-0/testFamily:testQualifier/1733221394632/Put/seqid=0 2024-12-03T10:23:50,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741858_1034 (size=5049) 2024-12-03T10:23:50,793 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 5cefb5eb6c037aa04998861c3c1edad3, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df, f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:50,993 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 5cefb5eb6c037aa04998861c3c1edad3, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df, f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:51,120 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/ns/68f3b9b5894c4e84be931a0acc64c821 2024-12-03T10:23:51,144 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3/.tmp/testFamily/3ae68a46c6fb4ca898368805b12faefe 2024-12-03T10:23:51,153 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3/.tmp/testFamily/3ae68a46c6fb4ca898368805b12faefe as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3/testFamily/3ae68a46c6fb4ca898368805b12faefe 2024-12-03T10:23:51,160 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3/testFamily/3ae68a46c6fb4ca898368805b12faefe, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:51,163 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5cefb5eb6c037aa04998861c3c1edad3 in 440ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:51,164 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/table/517ac34278b24c09b53049cab73b7105 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1733221370363/Put/seqid=0 2024-12-03T10:23:51,168 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScanner/5cefb5eb6c037aa04998861c3c1edad3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:51,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741859_1035 (size=6699) 2024-12-03T10:23:51,169 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:51,170 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5cefb5eb6c037aa04998861c3c1edad3: Waiting for close lock at 1733221430723Running coprocessor pre-close hooks at 1733221430723Disabling compacts and flushes for region at 1733221430723Disabling writes for close at 1733221430723Obtaining lock to block concurrent updates at 1733221430723Preparing flush snapshotting stores in 5cefb5eb6c037aa04998861c3c1edad3 at 1733221430723Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221430723Flushing stores of testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. at 1733221430724 (+1 ms)Flushing 5cefb5eb6c037aa04998861c3c1edad3/testFamily: creating writer at 1733221430724Flushing 5cefb5eb6c037aa04998861c3c1edad3/testFamily: appending metadata at 1733221430738 (+14 ms)Flushing 5cefb5eb6c037aa04998861c3c1edad3/testFamily: closing flushed file at 1733221430738Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22ea68b8: reopening flushed file at 1733221431151 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 5cefb5eb6c037aa04998861c3c1edad3 in 440ms, sequenceid=8, compaction requested=false at 1733221431163 (+12 ms)Writing region close event to WAL at 1733221431164 (+1 ms)Running coprocessor post-close hooks at 1733221431169 (+5 ms)Closed at 1733221431169 2024-12-03T10:23:51,170 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1733221393591.5cefb5eb6c037aa04998861c3c1edad3. 2024-12-03T10:23:51,170 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f69189b7b7330efefa5b500ab3856ac3, disabling compactions & flushes 2024-12-03T10:23:51,170 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:51,170 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:51,170 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. after waiting 0 ms 2024-12-03T10:23:51,170 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:51,170 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f69189b7b7330efefa5b500ab3856ac3 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:51,185 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3/.tmp/testFamily/846d63ce52704bc9b39748fc4b2af583 is 61, key is row-0/testFamily:testQualifier/1733221427089/Put/seqid=0 2024-12-03T10:23:51,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741860_1036 (size=5049) 2024-12-03T10:23:51,193 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-12-03T10:23:51,193 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1325): Online Regions={f69189b7b7330efefa5b500ab3856ac3=testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., 1588230740=hbase:meta,,1.1588230740, 34d30e8292e4b552cdf4f07ecdd16178=testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178., e38f183a25993c2b5fc891f9fc88f9df=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., df84c9bc6c4e06a825ca56632528287e=testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e., c2af42950dba1c869d770f12c3be50ef=hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef., 2eae6edbd9830e5528c525bf3f1f2923=testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923., 6f6d5ad9ad336e2b65dfd8227b7adcf7=hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., 80b17e09311ca90d67a44c0da1802b88=hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.} 2024-12-03T10:23:51,194 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df, f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:51,394 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df, f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:51,573 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/table/517ac34278b24c09b53049cab73b7105 2024-12-03T10:23:51,587 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/info/3e64c4e4cbde49aabc500540346288d2 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/info/3e64c4e4cbde49aabc500540346288d2 2024-12-03T10:23:51,591 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3/.tmp/testFamily/846d63ce52704bc9b39748fc4b2af583 2024-12-03T10:23:51,594 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df, f69189b7b7330efefa5b500ab3856ac3 2024-12-03T10:23:51,597 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/info/3e64c4e4cbde49aabc500540346288d2, entries=100, sequenceid=56, filesize=20.1 K 2024-12-03T10:23:51,598 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/ns/68f3b9b5894c4e84be931a0acc64c821 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/ns/68f3b9b5894c4e84be931a0acc64c821 2024-12-03T10:23:51,600 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3/.tmp/testFamily/846d63ce52704bc9b39748fc4b2af583 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3/testFamily/846d63ce52704bc9b39748fc4b2af583 2024-12-03T10:23:51,607 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3/testFamily/846d63ce52704bc9b39748fc4b2af583, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:51,607 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/ns/68f3b9b5894c4e84be931a0acc64c821, entries=2, sequenceid=56, filesize=5.0 K 2024-12-03T10:23:51,608 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f69189b7b7330efefa5b500ab3856ac3 in 438ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:51,609 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/.tmp/table/517ac34278b24c09b53049cab73b7105 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/table/517ac34278b24c09b53049cab73b7105 2024-12-03T10:23:51,613 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnOpenScannerAsync/f69189b7b7330efefa5b500ab3856ac3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:51,614 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:51,614 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f69189b7b7330efefa5b500ab3856ac3: Waiting for close lock at 1733221431170Running coprocessor pre-close hooks at 1733221431170Disabling compacts and flushes for region at 1733221431170Disabling writes for close at 1733221431170Obtaining lock to block concurrent updates at 1733221431170Preparing flush snapshotting stores in f69189b7b7330efefa5b500ab3856ac3 at 1733221431170Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221431170Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. at 1733221431171 (+1 ms)Flushing f69189b7b7330efefa5b500ab3856ac3/testFamily: creating writer at 1733221431171Flushing f69189b7b7330efefa5b500ab3856ac3/testFamily: appending metadata at 1733221431185 (+14 ms)Flushing f69189b7b7330efefa5b500ab3856ac3/testFamily: closing flushed file at 1733221431185Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34a6c96b: reopening flushed file at 1733221431599 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f69189b7b7330efefa5b500ab3856ac3 in 438ms, sequenceid=8, compaction requested=false at 1733221431608 (+9 ms)Writing region close event to WAL at 1733221431609 (+1 ms)Running coprocessor post-close hooks at 1733221431614 (+5 ms)Closed at 1733221431614 2024-12-03T10:23:51,615 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1733221426058.f69189b7b7330efefa5b500ab3856ac3. 2024-12-03T10:23:51,615 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34d30e8292e4b552cdf4f07ecdd16178, disabling compactions & flushes 2024-12-03T10:23:51,615 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:51,615 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:51,615 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. after waiting 0 ms 2024-12-03T10:23:51,615 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:51,615 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 34d30e8292e4b552cdf4f07ecdd16178 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:51,618 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/table/517ac34278b24c09b53049cab73b7105, entries=20, sequenceid=56, filesize=6.5 K 2024-12-03T10:23:51,620 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1428ms, sequenceid=56, compaction requested=false 2024-12-03T10:23:51,625 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-12-03T10:23:51,626 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T10:23:51,626 INFO [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T10:23:51,626 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733221430191Running coprocessor pre-close hooks at 1733221430191Disabling compacts and flushes for region at 1733221430191Disabling writes for close at 1733221430191Obtaining lock to block concurrent updates at 1733221430193 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1733221430193Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1733221430199 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733221430201 (+2 ms)Flushing 1588230740/info: creating writer at 1733221430202 (+1 ms)Flushing 1588230740/info: appending metadata at 1733221430246 (+44 ms)Flushing 1588230740/info: closing flushed file at 1733221430246Flushing 1588230740/ns: creating writer at 1733221430696 (+450 ms)Flushing 1588230740/ns: appending metadata at 1733221430712 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733221430713 (+1 ms)Flushing 1588230740/table: creating writer at 1733221431139 (+426 ms)Flushing 1588230740/table: appending metadata at 1733221431163 (+24 ms)Flushing 1588230740/table: closing flushed file at 1733221431163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a90c771: reopening flushed file at 1733221431586 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b4932fa: reopening flushed file at 1733221431597 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46b51bbe: reopening flushed file at 1733221431607 (+10 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1428ms, sequenceid=56, compaction requested=false at 1733221431620 (+13 ms)Writing region close event to WAL at 1733221431621 (+1 ms)Running coprocessor post-close hooks at 1733221431626 (+5 ms)Closed at 1733221431626 2024-12-03T10:23:51,627 DEBUG [RS_CLOSE_META-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T10:23:51,632 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178/.tmp/testFamily/1c0c2f2db74644e79cec24edad22285d is 61, key is row-0/testFamily:testQualifier/1733221406829/Put/seqid=0 2024-12-03T10:23:51,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741861_1037 (size=5049) 2024-12-03T10:23:51,795 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:51,995 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 34d30e8292e4b552cdf4f07ecdd16178, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:52,038 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178/.tmp/testFamily/1c0c2f2db74644e79cec24edad22285d 2024-12-03T10:23:52,049 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178/.tmp/testFamily/1c0c2f2db74644e79cec24edad22285d as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178/testFamily/1c0c2f2db74644e79cec24edad22285d 2024-12-03T10:23:52,058 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178/testFamily/1c0c2f2db74644e79cec24edad22285d, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:52,060 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 34d30e8292e4b552cdf4f07ecdd16178 in 444ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:52,065 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNext/34d30e8292e4b552cdf4f07ecdd16178/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:52,066 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:52,066 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34d30e8292e4b552cdf4f07ecdd16178: Waiting for close lock at 1733221431615Running coprocessor pre-close hooks at 1733221431615Disabling compacts and flushes for region at 1733221431615Disabling writes for close at 1733221431615Obtaining lock to block concurrent updates at 1733221431615Preparing flush snapshotting stores in 34d30e8292e4b552cdf4f07ecdd16178 at 1733221431615Finished memstore snapshotting testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221431615Flushing stores of testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. at 1733221431616 (+1 ms)Flushing 34d30e8292e4b552cdf4f07ecdd16178/testFamily: creating writer at 1733221431616Flushing 34d30e8292e4b552cdf4f07ecdd16178/testFamily: appending metadata at 1733221431632 (+16 ms)Flushing 34d30e8292e4b552cdf4f07ecdd16178/testFamily: closing flushed file at 1733221431632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d851cf8: reopening flushed file at 1733221432047 (+415 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 34d30e8292e4b552cdf4f07ecdd16178 in 444ms, sequenceid=8, compaction requested=false at 1733221432060 (+13 ms)Writing region close event to WAL at 1733221432060Running coprocessor post-close hooks at 1733221432066 (+6 ms)Closed at 1733221432066 2024-12-03T10:23:52,066 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1733221403784.34d30e8292e4b552cdf4f07ecdd16178. 2024-12-03T10:23:52,066 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e38f183a25993c2b5fc891f9fc88f9df, disabling compactions & flushes 2024-12-03T10:23:52,066 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:52,066 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:52,066 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. after waiting 0 ms 2024-12-03T10:23:52,066 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:52,066 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e38f183a25993c2b5fc891f9fc88f9df 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:52,080 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df/.tmp/testFamily/5d812e2ed19d4354a48839dda185b9a1 is 61, key is row-0/testFamily:testQualifier/1733221412925/Put/seqid=0 2024-12-03T10:23:52,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741862_1038 (size=5049) 2024-12-03T10:23:52,196 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-12-03T10:23:52,196 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1325): Online Regions={e38f183a25993c2b5fc891f9fc88f9df=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., df84c9bc6c4e06a825ca56632528287e=testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e., c2af42950dba1c869d770f12c3be50ef=hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef., 2eae6edbd9830e5528c525bf3f1f2923=testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923., 6f6d5ad9ad336e2b65dfd8227b7adcf7=hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., 80b17e09311ca90d67a44c0da1802b88=hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.} 2024-12-03T10:23:52,196 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:52,331 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T10:23:52,396 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e, e38f183a25993c2b5fc891f9fc88f9df 2024-12-03T10:23:52,486 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df/.tmp/testFamily/5d812e2ed19d4354a48839dda185b9a1 2024-12-03T10:23:52,495 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df/.tmp/testFamily/5d812e2ed19d4354a48839dda185b9a1 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df/testFamily/5d812e2ed19d4354a48839dda185b9a1 2024-12-03T10:23:52,503 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df/testFamily/5d812e2ed19d4354a48839dda185b9a1, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:52,504 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e38f183a25993c2b5fc891f9fc88f9df in 438ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:52,509 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/e38f183a25993c2b5fc891f9fc88f9df/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:52,510 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:52,510 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e38f183a25993c2b5fc891f9fc88f9df: Waiting for close lock at 1733221432066Running coprocessor pre-close hooks at 1733221432066Disabling compacts and flushes for region at 1733221432066Disabling writes for close at 1733221432066Obtaining lock to block concurrent updates at 1733221432066Preparing flush snapshotting stores in e38f183a25993c2b5fc891f9fc88f9df at 1733221432066Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221432067 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. at 1733221432067Flushing e38f183a25993c2b5fc891f9fc88f9df/testFamily: creating writer at 1733221432067Flushing e38f183a25993c2b5fc891f9fc88f9df/testFamily: appending metadata at 1733221432080 (+13 ms)Flushing e38f183a25993c2b5fc891f9fc88f9df/testFamily: closing flushed file at 1733221432080Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cbb6d47: reopening flushed file at 1733221432494 (+414 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for e38f183a25993c2b5fc891f9fc88f9df in 438ms, sequenceid=8, compaction requested=false at 1733221432504 (+10 ms)Writing region close event to WAL at 1733221432505 (+1 ms)Running coprocessor post-close hooks at 1733221432510 (+5 ms)Closed at 1733221432510 2024-12-03T10:23:52,510 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1733221409869.e38f183a25993c2b5fc891f9fc88f9df. 2024-12-03T10:23:52,510 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df84c9bc6c4e06a825ca56632528287e, disabling compactions & flushes 2024-12-03T10:23:52,510 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:52,511 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:52,511 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. after waiting 0 ms 2024-12-03T10:23:52,511 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:52,511 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing df84c9bc6c4e06a825ca56632528287e 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:52,526 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e/.tmp/testFamily/ca59cb5e6bfd4990a1e4eee868ecae40 is 61, key is row-0/testFamily:testQualifier/1733221400736/Put/seqid=0 2024-12-03T10:23:52,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741863_1039 (size=5049) 2024-12-03T10:23:52,597 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:52,797 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef, df84c9bc6c4e06a825ca56632528287e 2024-12-03T10:23:52,934 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e/.tmp/testFamily/ca59cb5e6bfd4990a1e4eee868ecae40 2024-12-03T10:23:52,947 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e/.tmp/testFamily/ca59cb5e6bfd4990a1e4eee868ecae40 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e/testFamily/ca59cb5e6bfd4990a1e4eee868ecae40 2024-12-03T10:23:52,955 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e/testFamily/ca59cb5e6bfd4990a1e4eee868ecae40, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:52,956 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for df84c9bc6c4e06a825ca56632528287e in 445ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:52,961 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testNormalScanTimeoutOnNextAsync/df84c9bc6c4e06a825ca56632528287e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:52,962 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:52,962 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df84c9bc6c4e06a825ca56632528287e: Waiting for close lock at 1733221432510Running coprocessor pre-close hooks at 1733221432510Disabling compacts and flushes for region at 1733221432510Disabling writes for close at 1733221432511 (+1 ms)Obtaining lock to block concurrent updates at 1733221432511Preparing flush snapshotting stores in df84c9bc6c4e06a825ca56632528287e at 1733221432511Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221432511Flushing stores of testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. at 1733221432512 (+1 ms)Flushing df84c9bc6c4e06a825ca56632528287e/testFamily: creating writer at 1733221432512Flushing df84c9bc6c4e06a825ca56632528287e/testFamily: appending metadata at 1733221432526 (+14 ms)Flushing df84c9bc6c4e06a825ca56632528287e/testFamily: closing flushed file at 1733221432526Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42a4dd4d: reopening flushed file at 1733221432946 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for df84c9bc6c4e06a825ca56632528287e in 445ms, sequenceid=8, compaction requested=false at 1733221432956 (+10 ms)Writing region close event to WAL at 1733221432957 (+1 ms)Running coprocessor post-close hooks at 1733221432962 (+5 ms)Closed at 1733221432962 2024-12-03T10:23:52,962 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1733221397674.df84c9bc6c4e06a825ca56632528287e. 2024-12-03T10:23:52,962 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c2af42950dba1c869d770f12c3be50ef, disabling compactions & flushes 2024-12-03T10:23:52,962 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:52,963 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:52,963 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. after waiting 0 ms 2024-12-03T10:23:52,963 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:52,963 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c2af42950dba1c869d770f12c3be50ef 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:52,979 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef/.tmp/testFamily/1326766dfdf94942996c193ff15f4471 is 61, key is row-0/testFamily:testQualifier/1733221384434/Put/seqid=0 2024-12-03T10:23:52,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741864_1040 (size=5049) 2024-12-03T10:23:52,983 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef/.tmp/testFamily/1326766dfdf94942996c193ff15f4471 2024-12-03T10:23:52,991 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef/.tmp/testFamily/1326766dfdf94942996c193ff15f4471 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef/testFamily/1326766dfdf94942996c193ff15f4471 2024-12-03T10:23:52,997 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88, c2af42950dba1c869d770f12c3be50ef 2024-12-03T10:23:52,999 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef/testFamily/1326766dfdf94942996c193ff15f4471, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:53,001 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c2af42950dba1c869d770f12c3be50ef in 37ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:53,006 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNextAsync/c2af42950dba1c869d770f12c3be50ef/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:53,006 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:53,007 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c2af42950dba1c869d770f12c3be50ef: Waiting for close lock at 1733221432962Running coprocessor pre-close hooks at 1733221432962Disabling compacts and flushes for region at 1733221432962Disabling writes for close at 1733221432963 (+1 ms)Obtaining lock to block concurrent updates at 1733221432963Preparing flush snapshotting stores in c2af42950dba1c869d770f12c3be50ef at 1733221432963Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221432963Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. at 1733221432964 (+1 ms)Flushing c2af42950dba1c869d770f12c3be50ef/testFamily: creating writer at 1733221432964Flushing c2af42950dba1c869d770f12c3be50ef/testFamily: appending metadata at 1733221432978 (+14 ms)Flushing c2af42950dba1c869d770f12c3be50ef/testFamily: closing flushed file at 1733221432979 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aba0f94: reopening flushed file at 1733221432990 (+11 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for c2af42950dba1c869d770f12c3be50ef in 37ms, sequenceid=8, compaction requested=false at 1733221433001 (+11 ms)Writing region close event to WAL at 1733221433001Running coprocessor post-close hooks at 1733221433006 (+5 ms)Closed at 1733221433006 2024-12-03T10:23:53,007 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1733221381384.c2af42950dba1c869d770f12c3be50ef. 2024-12-03T10:23:53,007 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2eae6edbd9830e5528c525bf3f1f2923, disabling compactions & flushes 2024-12-03T10:23:53,007 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:53,007 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:53,007 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. after waiting 0 ms 2024-12-03T10:23:53,007 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:53,007 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2eae6edbd9830e5528c525bf3f1f2923 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:53,026 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923/.tmp/testFamily/7de135584aa6487887d2846ac54ac30a is 61, key is row-0/testFamily:testQualifier/1733221393534/Put/seqid=0 2024-12-03T10:23:53,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741865_1041 (size=5049) 2024-12-03T10:23:53,198 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-12-03T10:23:53,198 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1325): Online Regions={2eae6edbd9830e5528c525bf3f1f2923=testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923., 6f6d5ad9ad336e2b65dfd8227b7adcf7=hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., 80b17e09311ca90d67a44c0da1802b88=hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.} 2024-12-03T10:23:53,198 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:53,399 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 2eae6edbd9830e5528c525bf3f1f2923, 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:53,433 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923/.tmp/testFamily/7de135584aa6487887d2846ac54ac30a 2024-12-03T10:23:53,448 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923/.tmp/testFamily/7de135584aa6487887d2846ac54ac30a as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923/testFamily/7de135584aa6487887d2846ac54ac30a 2024-12-03T10:23:53,456 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923/testFamily/7de135584aa6487887d2846ac54ac30a, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:53,457 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2eae6edbd9830e5528c525bf3f1f2923 in 450ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:53,462 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/default/testRetryOutOfOrderScannerNextException/2eae6edbd9830e5528c525bf3f1f2923/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:53,463 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:53,463 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2eae6edbd9830e5528c525bf3f1f2923: Waiting for close lock at 1733221433007Running coprocessor pre-close hooks at 1733221433007Disabling compacts and flushes for region at 1733221433007Disabling writes for close at 1733221433007Obtaining lock to block concurrent updates at 1733221433007Preparing flush snapshotting stores in 2eae6edbd9830e5528c525bf3f1f2923 at 1733221433007Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221433008 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. at 1733221433008Flushing 2eae6edbd9830e5528c525bf3f1f2923/testFamily: creating writer at 1733221433008Flushing 2eae6edbd9830e5528c525bf3f1f2923/testFamily: appending metadata at 1733221433025 (+17 ms)Flushing 2eae6edbd9830e5528c525bf3f1f2923/testFamily: closing flushed file at 1733221433025Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1280f83f: reopening flushed file at 1733221433447 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 2eae6edbd9830e5528c525bf3f1f2923 in 450ms, sequenceid=8, compaction requested=false at 1733221433457 (+10 ms)Writing region close event to WAL at 1733221433458 (+1 ms)Running coprocessor post-close hooks at 1733221433463 (+5 ms)Closed at 1733221433463 2024-12-03T10:23:53,463 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1733221390483.2eae6edbd9830e5528c525bf3f1f2923. 2024-12-03T10:23:53,463 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6f6d5ad9ad336e2b65dfd8227b7adcf7, disabling compactions & flushes 2024-12-03T10:23:53,463 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:23:53,463 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:23:53,464 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. after waiting 0 ms 2024-12-03T10:23:53,464 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:23:53,464 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6f6d5ad9ad336e2b65dfd8227b7adcf7 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:53,482 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7/.tmp/testFamily/2a253f2a39f74557b776ea8b3e7b4765 is 61, key is row-0/testFamily:testQualifier/1733221375336/Put/seqid=0 2024-12-03T10:23:53,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741866_1042 (size=5049) 2024-12-03T10:23:53,599 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:53,800 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 6f6d5ad9ad336e2b65dfd8227b7adcf7, 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:53,890 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7/.tmp/testFamily/2a253f2a39f74557b776ea8b3e7b4765 2024-12-03T10:23:53,904 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7/.tmp/testFamily/2a253f2a39f74557b776ea8b3e7b4765 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7/testFamily/2a253f2a39f74557b776ea8b3e7b4765 2024-12-03T10:23:53,911 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7/testFamily/2a253f2a39f74557b776ea8b3e7b4765, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:53,913 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6f6d5ad9ad336e2b65dfd8227b7adcf7 in 449ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:53,917 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnOpenScanner/6f6d5ad9ad336e2b65dfd8227b7adcf7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:53,918 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:23:53,918 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6f6d5ad9ad336e2b65dfd8227b7adcf7: Waiting for close lock at 1733221433463Running coprocessor pre-close hooks at 1733221433463Disabling compacts and flushes for region at 1733221433463Disabling writes for close at 1733221433464 (+1 ms)Obtaining lock to block concurrent updates at 1733221433464Preparing flush snapshotting stores in 6f6d5ad9ad336e2b65dfd8227b7adcf7 at 1733221433464Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221433464Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. at 1733221433465 (+1 ms)Flushing 6f6d5ad9ad336e2b65dfd8227b7adcf7/testFamily: creating writer at 1733221433465Flushing 6f6d5ad9ad336e2b65dfd8227b7adcf7/testFamily: appending metadata at 1733221433482 (+17 ms)Flushing 6f6d5ad9ad336e2b65dfd8227b7adcf7/testFamily: closing flushed file at 1733221433482Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dd1bd53: reopening flushed file at 1733221433903 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6f6d5ad9ad336e2b65dfd8227b7adcf7 in 449ms, sequenceid=8, compaction requested=false at 1733221433913 (+10 ms)Writing region close event to WAL at 1733221433913Running coprocessor post-close hooks at 1733221433918 (+5 ms)Closed at 1733221433918 2024-12-03T10:23:53,918 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1733221372286.6f6d5ad9ad336e2b65dfd8227b7adcf7. 2024-12-03T10:23:53,918 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 80b17e09311ca90d67a44c0da1802b88, disabling compactions & flushes 2024-12-03T10:23:53,918 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:53,919 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:53,919 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. after waiting 0 ms 2024-12-03T10:23:53,919 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:53,919 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 80b17e09311ca90d67a44c0da1802b88 1/1 column families, dataSize=228 B heapSize=800 B 2024-12-03T10:23:53,934 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88/.tmp/testFamily/e8a969c723544f238ef99560a364531c is 61, key is row-0/testFamily:testQualifier/1733221420014/Put/seqid=0 2024-12-03T10:23:53,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741867_1043 (size=5049) 2024-12-03T10:23:54,000 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:54,200 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T10:23:54,200 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1325): Online Regions={80b17e09311ca90d67a44c0da1802b88=hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88.} 2024-12-03T10:23:54,201 DEBUG [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1351): Waiting on 80b17e09311ca90d67a44c0da1802b88 2024-12-03T10:23:54,341 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88/.tmp/testFamily/e8a969c723544f238ef99560a364531c 2024-12-03T10:23:54,356 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88/.tmp/testFamily/e8a969c723544f238ef99560a364531c as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88/testFamily/e8a969c723544f238ef99560a364531c 2024-12-03T10:23:54,363 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88/testFamily/e8a969c723544f238ef99560a364531c, entries=4, sequenceid=8, filesize=4.9 K 2024-12-03T10:23:54,364 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 80b17e09311ca90d67a44c0da1802b88 in 445ms, sequenceid=8, compaction requested=false 2024-12-03T10:23:54,369 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/data/hbase/testMetaScanTimeoutOnNext/80b17e09311ca90d67a44c0da1802b88/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-12-03T10:23:54,370 INFO [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:54,370 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 80b17e09311ca90d67a44c0da1802b88: Waiting for close lock at 1733221433918Running coprocessor pre-close hooks at 1733221433918Disabling compacts and flushes for region at 1733221433918Disabling writes for close at 1733221433919 (+1 ms)Obtaining lock to block concurrent updates at 1733221433919Preparing flush snapshotting stores in 80b17e09311ca90d67a44c0da1802b88 at 1733221433919Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1733221433919Flushing stores of hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. at 1733221433920 (+1 ms)Flushing 80b17e09311ca90d67a44c0da1802b88/testFamily: creating writer at 1733221433920Flushing 80b17e09311ca90d67a44c0da1802b88/testFamily: appending metadata at 1733221433933 (+13 ms)Flushing 80b17e09311ca90d67a44c0da1802b88/testFamily: closing flushed file at 1733221433933Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11f6e05: reopening flushed file at 1733221434354 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 80b17e09311ca90d67a44c0da1802b88 in 445ms, sequenceid=8, compaction requested=false at 1733221434364 (+10 ms)Writing region close event to WAL at 1733221434365 (+1 ms)Running coprocessor post-close hooks at 1733221434370 (+5 ms)Closed at 1733221434370 2024-12-03T10:23:54,370 DEBUG [RS_CLOSE_REGION-regionserver/a78f5601770a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1733221418973.80b17e09311ca90d67a44c0da1802b88. 2024-12-03T10:23:54,401 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(976): stopping server a78f5601770a,38085,1733221335758; all regions closed. 2024-12-03T10:23:54,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741834_1010 (size=19479) 2024-12-03T10:23:54,413 DEBUG [RS:0;a78f5601770a:38085 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/oldWALs 2024-12-03T10:23:54,413 INFO [RS:0;a78f5601770a:38085 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a78f5601770a%2C38085%2C1733221335758.meta:.meta(num 1733221368688) 2024-12-03T10:23:54,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741833_1009 (size=20031) 2024-12-03T10:23:54,416 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/WALs/a78f5601770a,38085,1733221335758/a78f5601770a%2C38085%2C1733221335758.1733221368296 not finished, retry = 0 2024-12-03T10:23:54,519 DEBUG [RS:0;a78f5601770a:38085 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/oldWALs 2024-12-03T10:23:54,519 INFO [RS:0;a78f5601770a:38085 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a78f5601770a%2C38085%2C1733221335758:(num 1733221368296) 2024-12-03T10:23:54,519 DEBUG [RS:0;a78f5601770a:38085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T10:23:54,520 INFO [RS:0;a78f5601770a:38085 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T10:23:54,520 INFO [RS:0;a78f5601770a:38085 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T10:23:54,520 INFO [RS:0;a78f5601770a:38085 {}] hbase.ChoreService(370): Chore service for: regionserver/a78f5601770a:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T10:23:54,520 INFO [RS:0;a78f5601770a:38085 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T10:23:54,520 INFO [regionserver/a78f5601770a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T10:23:54,522 INFO [RS:0;a78f5601770a:38085 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38085 2024-12-03T10:23:54,551 INFO [regionserver/a78f5601770a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T10:23:54,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T10:23:54,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a78f5601770a,38085,1733221335758 2024-12-03T10:23:54,560 INFO [RS:0;a78f5601770a:38085 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T10:23:54,568 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a78f5601770a,38085,1733221335758] 2024-12-03T10:23:54,576 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a78f5601770a,38085,1733221335758 already deleted, retry=false 2024-12-03T10:23:54,576 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a78f5601770a,38085,1733221335758 expired; onlineServers=0 2024-12-03T10:23:54,577 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a78f5601770a,36731,1733221334944' ***** 2024-12-03T10:23:54,577 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T10:23:54,577 INFO [M:0;a78f5601770a:36731 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T10:23:54,578 INFO [M:0;a78f5601770a:36731 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T10:23:54,578 DEBUG [M:0;a78f5601770a:36731 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T10:23:54,578 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T10:23:54,578 DEBUG [M:0;a78f5601770a:36731 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T10:23:54,579 DEBUG [master/a78f5601770a:0:becomeActiveMaster-HFileCleaner.large.0-1733221338104 {}] cleaner.HFileCleaner(306): Exit Thread[master/a78f5601770a:0:becomeActiveMaster-HFileCleaner.large.0-1733221338104,5,FailOnTimeoutGroup] 2024-12-03T10:23:54,579 DEBUG [master/a78f5601770a:0:becomeActiveMaster-HFileCleaner.small.0-1733221338105 {}] cleaner.HFileCleaner(306): Exit Thread[master/a78f5601770a:0:becomeActiveMaster-HFileCleaner.small.0-1733221338105,5,FailOnTimeoutGroup] 2024-12-03T10:23:54,579 INFO [M:0;a78f5601770a:36731 {}] hbase.ChoreService(370): Chore service for: master/a78f5601770a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T10:23:54,580 INFO [M:0;a78f5601770a:36731 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T10:23:54,580 DEBUG [M:0;a78f5601770a:36731 {}] master.HMaster(1795): Stopping service threads 2024-12-03T10:23:54,580 INFO [M:0;a78f5601770a:36731 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T10:23:54,581 INFO [M:0;a78f5601770a:36731 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T10:23:54,582 INFO [M:0;a78f5601770a:36731 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T10:23:54,583 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T10:23:54,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T10:23:54,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T10:23:54,589 DEBUG [M:0;a78f5601770a:36731 {}] zookeeper.ZKUtil(347): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T10:23:54,589 WARN [M:0;a78f5601770a:36731 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T10:23:54,590 INFO [M:0;a78f5601770a:36731 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/.lastflushedseqids 2024-12-03T10:23:54,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741868_1044 (size=712) 2024-12-03T10:23:54,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T10:23:54,669 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38085-0x1019c006e650001, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T10:23:54,670 INFO [RS:0;a78f5601770a:38085 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T10:23:54,670 INFO [RS:0;a78f5601770a:38085 {}] regionserver.HRegionServer(1031): Exiting; stopping=a78f5601770a,38085,1733221335758; zookeeper connection closed. 2024-12-03T10:23:54,671 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b8e414c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b8e414c 2024-12-03T10:23:54,672 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-03T10:23:55,004 INFO [M:0;a78f5601770a:36731 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T10:23:55,004 INFO [M:0;a78f5601770a:36731 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T10:23:55,005 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T10:23:55,005 INFO [M:0;a78f5601770a:36731 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T10:23:55,005 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T10:23:55,005 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T10:23:55,005 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T10:23:55,006 INFO [M:0;a78f5601770a:36731 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.05 KB heapSize=191.10 KB 2024-12-03T10:23:55,028 DEBUG [M:0;a78f5601770a:36731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6bc77573b304ee698839aa96d04f12b is 82, key is hbase:meta,,1/info:regioninfo/1733221368771/Put/seqid=0 2024-12-03T10:23:55,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741869_1045 (size=5672) 2024-12-03T10:23:55,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-12-03T10:23:55,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-12-03T10:23:55,307 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-12-03T10:23:55,435 INFO [M:0;a78f5601770a:36731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6bc77573b304ee698839aa96d04f12b 2024-12-03T10:23:55,468 DEBUG [M:0;a78f5601770a:36731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b151c9b445d8449cb84f3b5d7aa0f287 is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733221370370/Put/seqid=0 2024-12-03T10:23:55,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741870_1046 (size=14067) 2024-12-03T10:23:55,876 INFO [M:0;a78f5601770a:36731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.50 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b151c9b445d8449cb84f3b5d7aa0f287 2024-12-03T10:23:55,889 INFO [M:0;a78f5601770a:36731 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b151c9b445d8449cb84f3b5d7aa0f287 2024-12-03T10:23:55,906 DEBUG [M:0;a78f5601770a:36731 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa7ab33561b74855b1aa74f2a289019f is 69, key is a78f5601770a,38085,1733221335758/rs:state/1733221338151/Put/seqid=0 2024-12-03T10:23:55,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741871_1047 (size=5156) 2024-12-03T10:23:56,313 INFO [M:0;a78f5601770a:36731 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa7ab33561b74855b1aa74f2a289019f 2024-12-03T10:23:56,337 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f6bc77573b304ee698839aa96d04f12b as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f6bc77573b304ee698839aa96d04f12b 2024-12-03T10:23:56,344 INFO [M:0;a78f5601770a:36731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f6bc77573b304ee698839aa96d04f12b, entries=8, sequenceid=319, filesize=5.5 K 2024-12-03T10:23:56,345 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b151c9b445d8449cb84f3b5d7aa0f287 as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b151c9b445d8449cb84f3b5d7aa0f287 2024-12-03T10:23:56,351 INFO [M:0;a78f5601770a:36731 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b151c9b445d8449cb84f3b5d7aa0f287 2024-12-03T10:23:56,351 INFO [M:0;a78f5601770a:36731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b151c9b445d8449cb84f3b5d7aa0f287, entries=33, sequenceid=319, filesize=13.7 K 2024-12-03T10:23:56,353 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/aa7ab33561b74855b1aa74f2a289019f as hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aa7ab33561b74855b1aa74f2a289019f 2024-12-03T10:23:56,359 INFO [M:0;a78f5601770a:36731 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35783/user/jenkins/test-data/d69d989d-f98b-14e3-8700-31fc2267afe2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/aa7ab33561b74855b1aa74f2a289019f, entries=1, sequenceid=319, filesize=5.0 K 2024-12-03T10:23:56,360 INFO [M:0;a78f5601770a:36731 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.05 KB/166967, heapSize ~190.80 KB/195384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1355ms, sequenceid=319, compaction requested=false 2024-12-03T10:23:56,362 INFO [M:0;a78f5601770a:36731 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T10:23:56,362 DEBUG [M:0;a78f5601770a:36731 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733221435004Disabling compacts and flushes for region at 1733221435004Disabling writes for close at 1733221435005 (+1 ms)Obtaining lock to block concurrent updates at 1733221435006 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733221435006Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166967, getHeapSize=195624, getOffHeapSize=0, getCellsCount=366 at 1733221435007 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733221435008 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733221435009 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733221435028 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733221435028Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733221435449 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733221435467 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733221435467Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733221435889 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733221435905 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733221435905Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f99ad9c: reopening flushed file at 1733221436336 (+431 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6051063a: reopening flushed file at 1733221436344 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a29b75f: reopening flushed file at 1733221436352 (+8 ms)Finished flush of dataSize ~163.05 KB/166967, heapSize ~190.80 KB/195384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1355ms, sequenceid=319, compaction requested=false at 1733221436360 (+8 ms)Writing region close event to WAL at 1733221436362 (+2 ms)Closed at 1733221436362 2024-12-03T10:23:56,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35377 is added to blk_1073741830_1006 (size=188309) 2024-12-03T10:23:56,365 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T10:23:56,366 INFO [M:0;a78f5601770a:36731 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T10:23:56,366 INFO [M:0;a78f5601770a:36731 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36731 2024-12-03T10:23:56,366 INFO [M:0;a78f5601770a:36731 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T10:23:56,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T10:23:56,484 INFO [M:0;a78f5601770a:36731 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T10:23:56,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36731-0x1019c006e650000, quorum=127.0.0.1:55781, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T10:23:56,489 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5589cdf5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T10:23:56,491 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@75f79e6f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T10:23:56,491 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T10:23:56,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1cc8666a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T10:23:56,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39e2c913{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/hadoop.log.dir/,STOPPED} 2024-12-03T10:23:56,494 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T10:23:56,494 WARN [BP-953270253-172.17.0.2-1733221331514 heartbeating to localhost/127.0.0.1:35783 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T10:23:56,494 WARN [BP-953270253-172.17.0.2-1733221331514 heartbeating to localhost/127.0.0.1:35783 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-953270253-172.17.0.2-1733221331514 (Datanode Uuid 144d94f9-5688-48b7-b8c6-5c34939ecb05) service to localhost/127.0.0.1:35783 2024-12-03T10:23:56,494 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T10:23:56,495 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/cluster_9091d149-19d1-21ab-6965-e2071c9733a5/data/data1/current/BP-953270253-172.17.0.2-1733221331514 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T10:23:56,495 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/cluster_9091d149-19d1-21ab-6965-e2071c9733a5/data/data2/current/BP-953270253-172.17.0.2-1733221331514 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T10:23:56,496 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T10:23:56,506 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21a9f2dc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T10:23:56,507 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b7efa4f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T10:23:56,507 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T10:23:56,507 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13f713ac{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T10:23:56,507 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72daf39c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/81fc391a-2cb2-8e03-05f8-2006760e6719/hadoop.log.dir/,STOPPED} 2024-12-03T10:23:56,516 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T10:23:56,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down