2024-11-24 19:55:41,148 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-24 19:55:41,164 main DEBUG Took 0.014981 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 19:55:41,165 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 19:55:41,165 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 19:55:41,166 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 19:55:41,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,174 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 19:55:41,186 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,189 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,190 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,192 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,193 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,194 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,195 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,195 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,196 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,196 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,197 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,198 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,198 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,199 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,199 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,200 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,200 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,201 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,201 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 19:55:41,202 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,202 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 19:55:41,204 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 19:55:41,206 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 19:55:41,208 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 19:55:41,208 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 19:55:41,210 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 19:55:41,211 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 19:55:41,221 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 19:55:41,223 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 19:55:41,225 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 19:55:41,225 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 19:55:41,225 main DEBUG createAppenders(={Console}) 2024-11-24 19:55:41,226 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-24 19:55:41,226 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-24 19:55:41,226 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-24 19:55:41,227 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 19:55:41,227 main DEBUG OutputStream closed 2024-11-24 19:55:41,227 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 19:55:41,228 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 19:55:41,228 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-24 19:55:41,298 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 19:55:41,301 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 19:55:41,302 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 19:55:41,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 19:55:41,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 19:55:41,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 19:55:41,305 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 19:55:41,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 19:55:41,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 19:55:41,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 19:55:41,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 19:55:41,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 19:55:41,308 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 19:55:41,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 19:55:41,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 19:55:41,310 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 19:55:41,310 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 19:55:41,311 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 19:55:41,313 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 19:55:41,313 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-24 19:55:41,313 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 19:55:41,314 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-24T19:55:41,327 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-24 19:55:41,329 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 19:55:41,330 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T19:55:41,571 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5 2024-11-24T19:55:41,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T19:55:41,603 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/cluster_ea2b57c6-c725-4370-9708-e0f92e318f26, deleteOnExit=true 2024-11-24T19:55:41,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T19:55:41,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/test.cache.data in system properties and HBase conf 2024-11-24T19:55:41,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T19:55:41,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/hadoop.log.dir in system properties and HBase conf 2024-11-24T19:55:41,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T19:55:41,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T19:55:41,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T19:55:41,702 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T19:55:41,897 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T19:55:41,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T19:55:41,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T19:55:41,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T19:55:41,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T19:55:41,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T19:55:41,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T19:55:41,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T19:55:41,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T19:55:41,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T19:55:41,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/nfs.dump.dir in system properties and HBase conf 2024-11-24T19:55:41,911 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/java.io.tmpdir in system properties and HBase conf 2024-11-24T19:55:41,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T19:55:41,912 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T19:55:41,918 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T19:55:43,143 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T19:55:43,217 INFO [Time-limited test {}] log.Log(170): Logging initialized @2684ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T19:55:43,287 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T19:55:43,353 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T19:55:43,375 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T19:55:43,375 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T19:55:43,377 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T19:55:43,391 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T19:55:43,394 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/hadoop.log.dir/,AVAILABLE} 2024-11-24T19:55:43,395 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T19:55:43,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/java.io.tmpdir/jetty-localhost-41223-hadoop-hdfs-3_4_1-tests_jar-_-any-9337225204845463866/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T19:55:43,590 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:41223} 2024-11-24T19:55:43,590 INFO [Time-limited test {}] server.Server(415): Started @3057ms 2024-11-24T19:55:44,084 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T19:55:44,091 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T19:55:44,092 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T19:55:44,092 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T19:55:44,092 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T19:55:44,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/hadoop.log.dir/,AVAILABLE} 2024-11-24T19:55:44,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T19:55:44,197 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@429b6cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/java.io.tmpdir/jetty-localhost-46237-hadoop-hdfs-3_4_1-tests_jar-_-any-15519951199055974920/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T19:55:44,198 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:46237} 2024-11-24T19:55:44,198 INFO [Time-limited test {}] server.Server(415): Started @3665ms 2024-11-24T19:55:44,253 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T19:55:45,149 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/cluster_ea2b57c6-c725-4370-9708-e0f92e318f26/data/data1/current/BP-2147221055-172.17.0.2-1732478142511/current, will proceed with Du for space computation calculation, 2024-11-24T19:55:45,149 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/cluster_ea2b57c6-c725-4370-9708-e0f92e318f26/data/data2/current/BP-2147221055-172.17.0.2-1732478142511/current, will proceed with Du for space computation calculation, 2024-11-24T19:55:45,174 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T19:55:45,216 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x87f80a9179f974a4 with lease ID 0x2d8345e2ad57abbd: Processing first storage report for DS-fa94a948-1900-456e-993d-46f9a450c95f from datanode DatanodeRegistration(127.0.0.1:39895, datanodeUuid=fc68c959-dc9b-48fd-b5bc-f305ebf6a986, infoPort=43723, infoSecurePort=0, ipcPort=36501, storageInfo=lv=-57;cid=testClusterID;nsid=1846447149;c=1732478142511) 2024-11-24T19:55:45,217 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87f80a9179f974a4 with lease ID 0x2d8345e2ad57abbd: from storage DS-fa94a948-1900-456e-993d-46f9a450c95f node DatanodeRegistration(127.0.0.1:39895, datanodeUuid=fc68c959-dc9b-48fd-b5bc-f305ebf6a986, infoPort=43723, infoSecurePort=0, ipcPort=36501, storageInfo=lv=-57;cid=testClusterID;nsid=1846447149;c=1732478142511), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T19:55:45,217 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x87f80a9179f974a4 with lease ID 0x2d8345e2ad57abbd: Processing first storage report for DS-ddae6dcc-8f1a-4e03-9144-770379b410d6 from datanode DatanodeRegistration(127.0.0.1:39895, datanodeUuid=fc68c959-dc9b-48fd-b5bc-f305ebf6a986, infoPort=43723, infoSecurePort=0, ipcPort=36501, storageInfo=lv=-57;cid=testClusterID;nsid=1846447149;c=1732478142511) 2024-11-24T19:55:45,218 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87f80a9179f974a4 with lease ID 0x2d8345e2ad57abbd: from storage DS-ddae6dcc-8f1a-4e03-9144-770379b410d6 node DatanodeRegistration(127.0.0.1:39895, datanodeUuid=fc68c959-dc9b-48fd-b5bc-f305ebf6a986, infoPort=43723, infoSecurePort=0, ipcPort=36501, storageInfo=lv=-57;cid=testClusterID;nsid=1846447149;c=1732478142511), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T19:55:45,270 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5 2024-11-24T19:55:45,332 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/cluster_ea2b57c6-c725-4370-9708-e0f92e318f26/zookeeper_0, clientPort=53455, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/cluster_ea2b57c6-c725-4370-9708-e0f92e318f26/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/cluster_ea2b57c6-c725-4370-9708-e0f92e318f26/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T19:55:45,341 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53455 2024-11-24T19:55:45,350 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T19:55:45,352 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T19:55:45,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741825_1001 (size=7) 2024-11-24T19:55:45,962 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f with version=8 2024-11-24T19:55:45,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/hbase-staging 2024-11-24T19:55:46,039 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T19:55:46,273 INFO [Time-limited test {}] client.ConnectionUtils(128): master/32a03c886245:0 server-side Connection retries=9 2024-11-24T19:55:46,281 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:55:46,282 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:55:46,287 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:55:46,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:55:46,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:55:46,406 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T19:55:46,454 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T19:55:46,462 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T19:55:46,465 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:55:46,485 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 98070 (auto-detected) 2024-11-24T19:55:46,486 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T19:55:46,503 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45559 2024-11-24T19:55:46,522 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45559 connecting to ZooKeeper ensemble=127.0.0.1:53455 2024-11-24T19:55:46,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:455590x0, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:55:46,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45559-0x1016fb425e70000 connected 2024-11-24T19:55:46,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T19:55:46,719 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T19:55:46,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:55:46,731 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f, hbase.cluster.distributed=false 2024-11-24T19:55:46,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:55:46,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45559 2024-11-24T19:55:46,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45559 2024-11-24T19:55:46,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45559 2024-11-24T19:55:46,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45559 2024-11-24T19:55:46,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45559 2024-11-24T19:55:46,860 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/32a03c886245:0 server-side Connection retries=9 2024-11-24T19:55:46,861 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:55:46,861 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T19:55:46,861 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T19:55:46,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T19:55:46,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T19:55:46,865 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T19:55:46,868 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T19:55:46,868 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38229 2024-11-24T19:55:46,870 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38229 connecting to ZooKeeper ensemble=127.0.0.1:53455 2024-11-24T19:55:46,872 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T19:55:46,875 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T19:55:46,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382290x0, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T19:55:46,907 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38229-0x1016fb425e70001 connected 2024-11-24T19:55:46,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:55:46,911 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T19:55:46,918 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T19:55:46,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T19:55:46,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T19:55:46,926 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38229 2024-11-24T19:55:46,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38229 2024-11-24T19:55:46,927 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38229 2024-11-24T19:55:46,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38229 2024-11-24T19:55:46,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38229 2024-11-24T19:55:46,944 DEBUG [M:0;32a03c886245:45559 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;32a03c886245:45559 2024-11-24T19:55:46,945 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/32a03c886245,45559,1732478146124 2024-11-24T19:55:46,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:55:46,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:55:46,958 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/32a03c886245,45559,1732478146124 2024-11-24T19:55:46,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T19:55:46,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:46,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:46,991 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T19:55:46,992 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/32a03c886245,45559,1732478146124 from backup master directory 2024-11-24T19:55:47,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:55:47,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/32a03c886245,45559,1732478146124 2024-11-24T19:55:47,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T19:55:47,003 WARN [master/32a03c886245:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:55:47,003 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=32a03c886245,45559,1732478146124 2024-11-24T19:55:47,005 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T19:55:47,006 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T19:55:47,057 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/hbase.id] with ID: af73521c-bb2e-43d4-806b-d7b76a077308 2024-11-24T19:55:47,057 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/.tmp/hbase.id 2024-11-24T19:55:47,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741826_1002 (size=42) 2024-11-24T19:55:47,468 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/.tmp/hbase.id]:[hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/hbase.id] 2024-11-24T19:55:47,520 INFO [master/32a03c886245:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T19:55:47,525 INFO [master/32a03c886245:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T19:55:47,543 INFO [master/32a03c886245:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-24T19:55:47,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:47,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:47,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741827_1003 (size=196) 2024-11-24T19:55:48,020 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:55:48,023 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T19:55:48,041 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T19:55:48,046 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:55:48,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741828_1004 (size=1189) 2024-11-24T19:55:48,106 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store 2024-11-24T19:55:48,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741829_1005 (size=34) 2024-11-24T19:55:48,140 INFO [master/32a03c886245:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T19:55:48,144 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:55:48,145 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T19:55:48,146 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:55:48,146 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:55:48,148 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T19:55:48,148 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:55:48,149 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:55:48,150 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732478148145Disabling compacts and flushes for region at 1732478148145Disabling writes for close at 1732478148148 (+3 ms)Writing region close event to WAL at 1732478148149 (+1 ms)Closed at 1732478148149 2024-11-24T19:55:48,153 WARN [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/.initializing 2024-11-24T19:55:48,154 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/WALs/32a03c886245,45559,1732478146124 2024-11-24T19:55:48,165 INFO [master/32a03c886245:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:55:48,184 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32a03c886245%2C45559%2C1732478146124, suffix=, logDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/WALs/32a03c886245,45559,1732478146124, archiveDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/oldWALs, maxLogs=10 2024-11-24T19:55:48,208 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/WALs/32a03c886245,45559,1732478146124/32a03c886245%2C45559%2C1732478146124.1732478148189, exclude list is [], retry=0 2024-11-24T19:55:48,227 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39895,DS-fa94a948-1900-456e-993d-46f9a450c95f,DISK] 2024-11-24T19:55:48,232 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T19:55:48,271 INFO [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/WALs/32a03c886245,45559,1732478146124/32a03c886245%2C45559%2C1732478146124.1732478148189 2024-11-24T19:55:48,273 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43723:43723)] 2024-11-24T19:55:48,273 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:55:48,274 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:55:48,278 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,280 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T19:55:48,384 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:48,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:55:48,392 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T19:55:48,397 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:48,399 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:55:48,399 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T19:55:48,405 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:48,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:55:48,408 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T19:55:48,413 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:48,414 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:55:48,415 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,420 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,422 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,433 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,436 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,441 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:55:48,447 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T19:55:48,455 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:55:48,457 INFO [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66857235, jitterRate=-0.003749564290046692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:55:48,468 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732478148298Initializing all the Stores at 1732478148302 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478148303 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478148306 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478148307 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478148307Cleaning up temporary data from old regions at 1732478148436 (+129 ms)Region opened successfully at 1732478148468 (+32 ms) 2024-11-24T19:55:48,470 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T19:55:48,524 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45ed60b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:55:48,571 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T19:55:48,592 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T19:55:48,592 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T19:55:48,600 INFO [master/32a03c886245:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T19:55:48,603 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 3 msec 2024-11-24T19:55:48,611 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-24T19:55:48,611 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T19:55:48,660 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T19:55:48,673 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T19:55:48,728 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T19:55:48,731 INFO [master/32a03c886245:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T19:55:48,734 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T19:55:48,743 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T19:55:48,749 INFO [master/32a03c886245:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T19:55:48,755 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T19:55:48,764 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T19:55:48,769 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T19:55:48,782 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T19:55:48,809 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T19:55:48,819 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T19:55:48,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T19:55:48,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T19:55:48,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:48,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:48,846 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=32a03c886245,45559,1732478146124, sessionid=0x1016fb425e70000, setting cluster-up flag (Was=false) 2024-11-24T19:55:48,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:48,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:48,906 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T19:55:48,908 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32a03c886245,45559,1732478146124 2024-11-24T19:55:48,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:48,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:55:48,960 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T19:55:48,962 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=32a03c886245,45559,1732478146124 2024-11-24T19:55:48,972 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T19:55:49,036 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(746): ClusterId : af73521c-bb2e-43d4-806b-d7b76a077308 2024-11-24T19:55:49,047 DEBUG [RS:0;32a03c886245:38229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T19:55:49,064 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T19:55:49,070 DEBUG [RS:0;32a03c886245:38229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T19:55:49,071 DEBUG [RS:0;32a03c886245:38229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T19:55:49,077 INFO [master/32a03c886245:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T19:55:49,082 DEBUG [RS:0;32a03c886245:38229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T19:55:49,083 DEBUG [RS:0;32a03c886245:38229 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edb43c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=32a03c886245/172.17.0.2:0 2024-11-24T19:55:49,086 INFO [master/32a03c886245:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T19:55:49,094 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 32a03c886245,45559,1732478146124 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T19:55:49,104 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:55:49,104 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:55:49,104 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:55:49,104 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;32a03c886245:38229 2024-11-24T19:55:49,105 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/32a03c886245:0, corePoolSize=5, maxPoolSize=5 2024-11-24T19:55:49,105 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/32a03c886245:0, corePoolSize=10, maxPoolSize=10 2024-11-24T19:55:49,105 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,105 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:55:49,105 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,108 INFO [RS:0;32a03c886245:38229 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T19:55:49,108 INFO [RS:0;32a03c886245:38229 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T19:55:49,109 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T19:55:49,112 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732478179112 2024-11-24T19:55:49,112 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(2659): reportForDuty to master=32a03c886245,45559,1732478146124 with port=38229, startcode=1732478146827 2024-11-24T19:55:49,114 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T19:55:49,114 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:55:49,114 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T19:55:49,115 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T19:55:49,119 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T19:55:49,119 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T19:55:49,120 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T19:55:49,120 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T19:55:49,120 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,122 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:49,122 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T19:55:49,125 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T19:55:49,126 DEBUG [RS:0;32a03c886245:38229 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T19:55:49,128 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T19:55:49,128 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T19:55:49,143 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T19:55:49,144 INFO [master/32a03c886245:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T19:55:49,153 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478149147,5,FailOnTimeoutGroup] 2024-11-24T19:55:49,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741831_1007 (size=1321) 2024-11-24T19:55:49,158 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478149153,5,FailOnTimeoutGroup] 2024-11-24T19:55:49,159 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,159 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T19:55:49,160 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,160 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,180 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50325, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T19:55:49,188 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45559 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 32a03c886245,38229,1732478146827 2024-11-24T19:55:49,191 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45559 {}] master.ServerManager(517): Registering regionserver=32a03c886245,38229,1732478146827 2024-11-24T19:55:49,208 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:55:49,208 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35739 2024-11-24T19:55:49,208 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T19:55:49,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:55:49,244 DEBUG [RS:0;32a03c886245:38229 {}] zookeeper.ZKUtil(111): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/32a03c886245,38229,1732478146827 2024-11-24T19:55:49,245 WARN [RS:0;32a03c886245:38229 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T19:55:49,245 INFO [RS:0;32a03c886245:38229 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:55:49,245 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/WALs/32a03c886245,38229,1732478146827 2024-11-24T19:55:49,248 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [32a03c886245,38229,1732478146827] 2024-11-24T19:55:49,275 INFO [RS:0;32a03c886245:38229 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T19:55:49,292 INFO [RS:0;32a03c886245:38229 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T19:55:49,297 INFO [RS:0;32a03c886245:38229 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T19:55:49,298 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,299 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T19:55:49,306 INFO [RS:0;32a03c886245:38229 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T19:55:49,307 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,308 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,308 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,308 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,308 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,308 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,308 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/32a03c886245:0, corePoolSize=2, maxPoolSize=2 2024-11-24T19:55:49,309 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,309 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,309 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,309 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,309 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,309 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/32a03c886245:0, corePoolSize=1, maxPoolSize=1 2024-11-24T19:55:49,309 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:55:49,309 DEBUG [RS:0;32a03c886245:38229 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/32a03c886245:0, corePoolSize=3, maxPoolSize=3 2024-11-24T19:55:49,311 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,311 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,311 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,311 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,311 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,312 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,38229,1732478146827-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:55:49,336 INFO [RS:0;32a03c886245:38229 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T19:55:49,338 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,38229,1732478146827-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,338 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,339 INFO [RS:0;32a03c886245:38229 {}] regionserver.Replication(171): 32a03c886245,38229,1732478146827 started 2024-11-24T19:55:49,363 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:55:49,364 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1482): Serving as 32a03c886245,38229,1732478146827, RpcServer on 32a03c886245/172.17.0.2:38229, sessionid=0x1016fb425e70001 2024-11-24T19:55:49,364 DEBUG [RS:0;32a03c886245:38229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T19:55:49,365 DEBUG [RS:0;32a03c886245:38229 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 32a03c886245,38229,1732478146827 2024-11-24T19:55:49,365 DEBUG [RS:0;32a03c886245:38229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,38229,1732478146827' 2024-11-24T19:55:49,365 DEBUG [RS:0;32a03c886245:38229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T19:55:49,366 DEBUG [RS:0;32a03c886245:38229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T19:55:49,367 DEBUG [RS:0;32a03c886245:38229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T19:55:49,367 DEBUG [RS:0;32a03c886245:38229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T19:55:49,367 DEBUG [RS:0;32a03c886245:38229 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 32a03c886245,38229,1732478146827 2024-11-24T19:55:49,367 DEBUG [RS:0;32a03c886245:38229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '32a03c886245,38229,1732478146827' 2024-11-24T19:55:49,368 DEBUG [RS:0;32a03c886245:38229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T19:55:49,368 DEBUG [RS:0;32a03c886245:38229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T19:55:49,369 DEBUG [RS:0;32a03c886245:38229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T19:55:49,369 INFO [RS:0;32a03c886245:38229 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T19:55:49,369 INFO [RS:0;32a03c886245:38229 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T19:55:49,559 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T19:55:49,559 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:55:49,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741832_1008 (size=32) 2024-11-24T19:55:49,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:55:49,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T19:55:49,977 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T19:55:49,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:49,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:55:49,979 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T19:55:49,982 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T19:55:49,982 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:49,984 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:55:49,984 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T19:55:49,987 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T19:55:49,987 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:49,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:55:49,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T19:55:49,991 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T19:55:49,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:55:49,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:55:49,992 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T19:55:49,994 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740 2024-11-24T19:55:49,995 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740 2024-11-24T19:55:49,999 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T19:55:49,999 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T19:55:50,000 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:55:50,004 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T19:55:50,009 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:55:50,010 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62623123, jitterRate=-0.06684274971485138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:55:50,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732478149971Initializing all the Stores at 1732478149973 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478149973Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478149973Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478149973Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478149973Cleaning up temporary data from old regions at 1732478149999 (+26 ms)Region opened successfully at 1732478150013 (+14 ms) 2024-11-24T19:55:50,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T19:55:50,013 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T19:55:50,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T19:55:50,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T19:55:50,014 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T19:55:50,015 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T19:55:50,015 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732478150013Disabling compacts and flushes for region at 1732478150013Disabling writes for close at 1732478150014 (+1 ms)Writing region close event to WAL at 1732478150015 (+1 ms)Closed at 1732478150015 2024-11-24T19:55:50,018 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:55:50,019 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T19:55:50,025 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T19:55:50,032 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T19:55:50,036 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T19:55:50,187 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:51,187 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:52,188 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:53,189 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:54,190 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:55,191 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:56,192 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:56,452 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T19:55:56,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T19:55:57,193 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:58,194 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:55:59,195 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:00,195 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:01,196 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:02,197 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:03,198 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:04,198 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:05,199 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:06,200 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:07,201 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:08,202 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:09,202 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:10,203 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:11,204 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:12,205 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:13,206 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:14,206 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:15,207 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:15,267 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T19:56:16,208 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:17,209 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:18,210 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:19,210 WARN [32a03c886245:45559 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T19:56:19,376 INFO [RS:0;32a03c886245:38229 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T19:56:19,383 INFO [RS:0;32a03c886245:38229 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32a03c886245%2C38229%2C1732478146827, suffix=, logDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/WALs/32a03c886245,38229,1732478146827, archiveDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/oldWALs, maxLogs=32 2024-11-24T19:56:19,400 DEBUG [RS:0;32a03c886245:38229 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/WALs/32a03c886245,38229,1732478146827/32a03c886245%2C38229%2C1732478146827.1732478179387, exclude list is [], retry=0 2024-11-24T19:56:19,404 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39895,DS-fa94a948-1900-456e-993d-46f9a450c95f,DISK] 2024-11-24T19:56:19,408 INFO [RS:0;32a03c886245:38229 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/WALs/32a03c886245,38229,1732478146827/32a03c886245%2C38229%2C1732478146827.1732478179387 2024-11-24T19:56:19,408 DEBUG [RS:0;32a03c886245:38229 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43723:43723)] 2024-11-24T19:56:19,464 DEBUG [32a03c886245:45559 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T19:56:19,477 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:19,483 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a03c886245,38229,1732478146827, state=OPENING 2024-11-24T19:56:19,524 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T19:56:19,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:56:19,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:56:19,534 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:56:19,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:56:19,538 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T19:56:19,541 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:56:19,714 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T19:56:19,717 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55475, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T19:56:19,728 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T19:56:19,728 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T19:56:19,729 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T19:56:19,732 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=32a03c886245%2C38229%2C1732478146827.meta, suffix=.meta, logDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/WALs/32a03c886245,38229,1732478146827, archiveDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/oldWALs, maxLogs=32 2024-11-24T19:56:19,747 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/WALs/32a03c886245,38229,1732478146827/32a03c886245%2C38229%2C1732478146827.meta.1732478179734.meta, exclude list is [], retry=0 2024-11-24T19:56:19,751 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39895,DS-fa94a948-1900-456e-993d-46f9a450c95f,DISK] 2024-11-24T19:56:19,755 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/WALs/32a03c886245,38229,1732478146827/32a03c886245%2C38229%2C1732478146827.meta.1732478179734.meta 2024-11-24T19:56:19,756 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43723:43723)] 2024-11-24T19:56:19,756 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:56:19,757 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T19:56:19,760 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T19:56:19,763 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T19:56:19,767 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T19:56:19,768 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:19,768 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T19:56:19,768 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T19:56:19,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T19:56:19,773 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T19:56:19,773 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:19,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:56:19,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T19:56:19,775 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T19:56:19,775 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:19,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:56:19,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T19:56:19,778 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T19:56:19,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:19,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:56:19,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T19:56:19,780 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T19:56:19,780 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:19,781 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T19:56:19,781 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T19:56:19,783 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740 2024-11-24T19:56:19,786 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740 2024-11-24T19:56:19,789 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T19:56:19,789 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T19:56:19,790 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T19:56:19,792 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T19:56:19,794 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70844422, jitterRate=0.05566415190696716}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T19:56:19,794 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T19:56:19,796 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732478179769Writing region info on filesystem at 1732478179769Initializing all the Stores at 1732478179770 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478179771 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478179771Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478179771Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732478179771Cleaning up temporary data from old regions at 1732478179789 (+18 ms)Running coprocessor post-open hooks at 1732478179794 (+5 ms)Region opened successfully at 1732478179796 (+2 ms) 2024-11-24T19:56:19,803 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732478179706 2024-11-24T19:56:19,813 DEBUG [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T19:56:19,814 INFO [RS_OPEN_META-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T19:56:19,815 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:19,818 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 32a03c886245,38229,1732478146827, state=OPEN 2024-11-24T19:56:19,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:56:19,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T19:56:19,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:56:19,866 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T19:56:19,867 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=32a03c886245,38229,1732478146827 2024-11-24T19:56:19,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T19:56:19,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=32a03c886245,38229,1732478146827 in 327 msec 2024-11-24T19:56:19,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T19:56:19,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 29.8520 sec 2024-11-24T19:56:19,883 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T19:56:19,883 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T19:56:19,901 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T19:56:19,902 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a03c886245,38229,1732478146827, seqNum=-1] 2024-11-24T19:56:19,919 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T19:56:19,922 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59145, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T19:56:19,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 30.9310 sec 2024-11-24T19:56:19,941 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732478179941, completionTime=-1 2024-11-24T19:56:19,945 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T19:56:19,945 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T19:56:19,971 INFO [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T19:56:19,971 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732478239971 2024-11-24T19:56:19,971 INFO [master/32a03c886245:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732478299971 2024-11-24T19:56:19,971 INFO [master/32a03c886245:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-24T19:56:19,973 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,45559,1732478146124-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T19:56:19,973 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,45559,1732478146124-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:56:19,973 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,45559,1732478146124-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:56:19,975 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-32a03c886245:45559, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:56:19,975 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T19:56:19,976 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T19:56:19,982 DEBUG [master/32a03c886245:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T19:56:20,002 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 32.998sec 2024-11-24T19:56:20,003 INFO [master/32a03c886245:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T19:56:20,005 INFO [master/32a03c886245:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T19:56:20,006 INFO [master/32a03c886245:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T19:56:20,006 INFO [master/32a03c886245:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T19:56:20,006 INFO [master/32a03c886245:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T19:56:20,007 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,45559,1732478146124-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T19:56:20,007 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,45559,1732478146124-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T19:56:20,015 DEBUG [master/32a03c886245:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T19:56:20,016 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T19:56:20,016 INFO [master/32a03c886245:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=32a03c886245,45559,1732478146124-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T19:56:20,077 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573d5973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:56:20,079 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T19:56:20,079 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T19:56:20,082 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a03c886245,45559,-1 for getting cluster id 2024-11-24T19:56:20,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T19:56:20,092 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'af73521c-bb2e-43d4-806b-d7b76a077308' 2024-11-24T19:56:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T19:56:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "af73521c-bb2e-43d4-806b-d7b76a077308" 2024-11-24T19:56:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34fc4220, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:56:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a03c886245,45559,-1] 2024-11-24T19:56:20,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T19:56:20,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:56:20,119 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T19:56:20,123 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1928876b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:56:20,123 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T19:56:20,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a03c886245,38229,1732478146827, seqNum=-1] 2024-11-24T19:56:20,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T19:56:20,134 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T19:56:20,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=32a03c886245,45559,1732478146124 2024-11-24T19:56:20,162 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2246aef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:56:20,162 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 32a03c886245,45559,-1 for getting cluster id 2024-11-24T19:56:20,162 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T19:56:20,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'af73521c-bb2e-43d4-806b-d7b76a077308' 2024-11-24T19:56:20,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T19:56:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "af73521c-bb2e-43d4-806b-d7b76a077308" 2024-11-24T19:56:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71e1532c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:56:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [32a03c886245,45559,-1] 2024-11-24T19:56:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T19:56:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:56:20,169 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T19:56:20,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1640eae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T19:56:20,184 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=220, ProcessCount=11, AvailableMemoryMB=10858 2024-11-24T19:56:20,191 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T19:56:20,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 32a03c886245,45559,1732478146124 2024-11-24T19:56:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7fb07f3e 2024-11-24T19:56:20,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T19:56:20,201 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T19:56:20,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:56:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-24T19:56:20,217 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:56:20,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-24T19:56:20,220 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:20,222 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:56:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:56:20,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741835_1011 (size=428) 2024-11-24T19:56:20,638 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b7380c45a53f728d75b96169ee5e67d, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:56:20,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741836_1012 (size=79) 2024-11-24T19:56:21,052 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:21,052 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 8b7380c45a53f728d75b96169ee5e67d, disabling compactions & flushes 2024-11-24T19:56:21,052 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:56:21,052 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:56:21,052 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. after waiting 0 ms 2024-11-24T19:56:21,053 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:56:21,053 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:56:21,053 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b7380c45a53f728d75b96169ee5e67d: Waiting for close lock at 1732478181052Disabling compacts and flushes for region at 1732478181052Disabling writes for close at 1732478181052Writing region close event to WAL at 1732478181053 (+1 ms)Closed at 1732478181053 2024-11-24T19:56:21,057 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:56:21,064 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732478181058"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478181058"}]},"ts":"1732478181058"} 2024-11-24T19:56:21,070 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:56:21,071 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:56:21,074 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478181072"}]},"ts":"1732478181072"} 2024-11-24T19:56:21,079 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-24T19:56:21,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=8b7380c45a53f728d75b96169ee5e67d, ASSIGN}] 2024-11-24T19:56:21,084 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=8b7380c45a53f728d75b96169ee5e67d, ASSIGN 2024-11-24T19:56:21,086 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=8b7380c45a53f728d75b96169ee5e67d, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:56:21,238 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8b7380c45a53f728d75b96169ee5e67d, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:56:21,241 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=8b7380c45a53f728d75b96169ee5e67d, ASSIGN because future has completed 2024-11-24T19:56:21,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b7380c45a53f728d75b96169ee5e67d, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:56:21,410 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:56:21,411 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b7380c45a53f728d75b96169ee5e67d, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:56:21,411 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,411 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:21,411 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,411 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,414 INFO [StoreOpener-8b7380c45a53f728d75b96169ee5e67d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,416 INFO [StoreOpener-8b7380c45a53f728d75b96169ee5e67d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b7380c45a53f728d75b96169ee5e67d columnFamilyName testFamily 2024-11-24T19:56:21,416 DEBUG [StoreOpener-8b7380c45a53f728d75b96169ee5e67d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:21,418 INFO [StoreOpener-8b7380c45a53f728d75b96169ee5e67d-1 {}] regionserver.HStore(327): Store=8b7380c45a53f728d75b96169ee5e67d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:56:21,418 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,419 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,420 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,420 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,421 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,423 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,427 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:56:21,428 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8b7380c45a53f728d75b96169ee5e67d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74360060, jitterRate=0.10805124044418335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:56:21,428 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:56:21,429 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8b7380c45a53f728d75b96169ee5e67d: Running coprocessor pre-open hook at 1732478181412Writing region info on filesystem at 1732478181412Initializing all the Stores at 1732478181413 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478181413Cleaning up temporary data from old regions at 1732478181421 (+8 ms)Running coprocessor post-open hooks at 1732478181428 (+7 ms)Region opened successfully at 1732478181429 (+1 ms) 2024-11-24T19:56:21,431 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d., pid=6, masterSystemTime=1732478181397 2024-11-24T19:56:21,434 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:56:21,434 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:56:21,435 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8b7380c45a53f728d75b96169ee5e67d, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:21,439 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b7380c45a53f728d75b96169ee5e67d, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:56:21,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T19:56:21,446 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b7380c45a53f728d75b96169ee5e67d, server=32a03c886245,38229,1732478146827 in 199 msec 2024-11-24T19:56:21,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T19:56:21,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=8b7380c45a53f728d75b96169ee5e67d, ASSIGN in 363 msec 2024-11-24T19:56:21,450 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:56:21,450 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478181450"}]},"ts":"1732478181450"} 2024-11-24T19:56:21,453 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-24T19:56:21,455 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:56:21,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2450 sec 2024-11-24T19:56:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T19:56:23,249 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-24T19:56:23,249 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-24T19:56:23,251 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:23,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-24T19:56:23,258 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:23,258 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-24T19:56:23,261 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T19:56:23,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=32a03c886245,38229,1732478146827, seqNum=-1] 2024-11-24T19:56:23,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T19:56:23,265 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T19:56:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:56:23,285 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:56:23,289 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-24T19:56:23,293 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-24T19:56:23,298 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-24T19:56:23,300 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-24T19:56:23,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3309366211027927032 2024-11-24T19:56:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40398 deadline: 1732478186299, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T19:56:23,330 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-24T19:56:23,331 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-24T19:56:23,333 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:23,336 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:23,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3309366211027927030 2024-11-24T19:56:23,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40398 deadline: 1732478186335, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T19:56:23,342 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:23,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3309366211027927029 2024-11-24T19:56:23,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40398 deadline: 1732478186342, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T19:56:23,347 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:23,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3309366211027927028 2024-11-24T19:56:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40398 deadline: 1732478186347, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T19:56:23,353 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:23,363 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1545303662_22 at /127.0.0.1:52594 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/32a03c886245:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=203 (was 220), ProcessCount=11 (was 11), AvailableMemoryMB=10829 (was 10858) 2024-11-24T19:56:23,371 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=203, ProcessCount=11, AvailableMemoryMB=10828 2024-11-24T19:56:23,375 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:56:23,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-24T19:56:23,379 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:56:23,379 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:23,380 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-24T19:56:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T19:56:23,382 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:56:23,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741837_1013 (size=414) 2024-11-24T19:56:23,795 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 52407b6249c71db9494e29d4d16771fd, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:56:23,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741838_1014 (size=65) 2024-11-24T19:56:24,206 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:24,206 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 52407b6249c71db9494e29d4d16771fd, disabling compactions & flushes 2024-11-24T19:56:24,206 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:56:24,206 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:56:24,206 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. after waiting 0 ms 2024-11-24T19:56:24,206 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:56:24,207 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:56:24,207 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 52407b6249c71db9494e29d4d16771fd: Waiting for close lock at 1732478184206Disabling compacts and flushes for region at 1732478184206Disabling writes for close at 1732478184206Writing region close event to WAL at 1732478184207 (+1 ms)Closed at 1732478184207 2024-11-24T19:56:24,210 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:56:24,210 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732478184210"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478184210"}]},"ts":"1732478184210"} 2024-11-24T19:56:24,214 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:56:24,217 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:56:24,217 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478184217"}]},"ts":"1732478184217"} 2024-11-24T19:56:24,221 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-24T19:56:24,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=52407b6249c71db9494e29d4d16771fd, ASSIGN}] 2024-11-24T19:56:24,223 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=52407b6249c71db9494e29d4d16771fd, ASSIGN 2024-11-24T19:56:24,225 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=52407b6249c71db9494e29d4d16771fd, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:56:24,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=52407b6249c71db9494e29d4d16771fd, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:24,387 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=52407b6249c71db9494e29d4d16771fd, ASSIGN because future has completed 2024-11-24T19:56:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T19:56:24,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52407b6249c71db9494e29d4d16771fd, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:56:24,549 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:56:24,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 52407b6249c71db9494e29d4d16771fd, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:56:24,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:24,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,550 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,553 INFO [StoreOpener-52407b6249c71db9494e29d4d16771fd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,555 INFO [StoreOpener-52407b6249c71db9494e29d4d16771fd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52407b6249c71db9494e29d4d16771fd columnFamilyName testFamily 2024-11-24T19:56:24,556 DEBUG [StoreOpener-52407b6249c71db9494e29d4d16771fd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:24,557 INFO [StoreOpener-52407b6249c71db9494e29d4d16771fd-1 {}] regionserver.HStore(327): Store=52407b6249c71db9494e29d4d16771fd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:56:24,557 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,560 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,560 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,564 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,568 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:56:24,569 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 52407b6249c71db9494e29d4d16771fd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69259836, jitterRate=0.03205198049545288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:56:24,569 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:56:24,570 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 52407b6249c71db9494e29d4d16771fd: Running coprocessor pre-open hook at 1732478184551Writing region info on filesystem at 1732478184551Initializing all the Stores at 1732478184552 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478184552Cleaning up temporary data from old regions at 1732478184560 (+8 ms)Running coprocessor post-open hooks at 1732478184569 (+9 ms)Region opened successfully at 1732478184570 (+1 ms) 2024-11-24T19:56:24,572 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd., pid=9, masterSystemTime=1732478184543 2024-11-24T19:56:24,576 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:56:24,576 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:56:24,578 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=52407b6249c71db9494e29d4d16771fd, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:24,582 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52407b6249c71db9494e29d4d16771fd, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:56:24,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T19:56:24,589 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 52407b6249c71db9494e29d4d16771fd, server=32a03c886245,38229,1732478146827 in 195 msec 2024-11-24T19:56:24,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T19:56:24,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=52407b6249c71db9494e29d4d16771fd, ASSIGN in 368 msec 2024-11-24T19:56:24,594 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:56:24,595 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478184594"}]},"ts":"1732478184594"} 2024-11-24T19:56:24,598 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-24T19:56:24,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:56:24,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2250 sec 2024-11-24T19:56:25,308 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:56:25,340 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-24T19:56:25,342 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T19:56:25,343 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-24T19:56:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T19:56:26,402 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-24T19:56:26,402 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-24T19:56:26,403 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:26,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-24T19:56:26,414 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:26,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-24T19:56:26,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:56:26,429 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:56:26,429 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-24T19:56:26,433 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-24T19:56:26,435 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-24T19:56:26,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T19:56:26,451 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T19:56:26,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T19:56:26,453 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T19:56:26,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-24T19:56:26,453 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T19:56:26,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-24T19:56:26,454 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-24T19:56:32,456 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd., hostname=32a03c886245,38229,1732478146827, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd., hostname=32a03c886245,38229,1732478146827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=32a03c886245:38229 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6019ms, rpcTimeout=5999ms 2024-11-24T19:56:32,457 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd., hostname=32a03c886245,38229,1732478146827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6019ms, rpcTimeout=5999ms 2024-11-24T19:56:32,457 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd., hostname=32a03c886245,38229,1732478146827, seqNum=2 from cache 2024-11-24T19:56:32,465 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=171 (was 203), ProcessCount=11 (was 11), AvailableMemoryMB=10745 (was 10828) 2024-11-24T19:56:32,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=171, ProcessCount=11, AvailableMemoryMB=10744 2024-11-24T19:56:32,476 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:56:32,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-24T19:56:32,480 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:56:32,480 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:32,480 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-24T19:56:32,481 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:56:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-24T19:56:32,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741839_1015 (size=412) 2024-11-24T19:56:32,899 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 566da3ef551e53b7c5fe23ca2c3e4c49, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:56:32,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741840_1016 (size=63) 2024-11-24T19:56:32,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-24T19:56:33,315 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:33,315 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 566da3ef551e53b7c5fe23ca2c3e4c49, disabling compactions & flushes 2024-11-24T19:56:33,315 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:56:33,316 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:56:33,316 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. after waiting 0 ms 2024-11-24T19:56:33,316 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:56:33,316 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:56:33,316 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 566da3ef551e53b7c5fe23ca2c3e4c49: Waiting for close lock at 1732478193315Disabling compacts and flushes for region at 1732478193315Disabling writes for close at 1732478193316 (+1 ms)Writing region close event to WAL at 1732478193316Closed at 1732478193316 2024-11-24T19:56:33,322 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:56:33,323 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732478193322"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478193322"}]},"ts":"1732478193322"} 2024-11-24T19:56:33,326 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:56:33,328 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:56:33,329 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478193328"}]},"ts":"1732478193328"} 2024-11-24T19:56:33,331 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-24T19:56:33,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=566da3ef551e53b7c5fe23ca2c3e4c49, ASSIGN}] 2024-11-24T19:56:33,333 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=566da3ef551e53b7c5fe23ca2c3e4c49, ASSIGN 2024-11-24T19:56:33,335 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=566da3ef551e53b7c5fe23ca2c3e4c49, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:56:33,486 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=566da3ef551e53b7c5fe23ca2c3e4c49, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:33,496 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=566da3ef551e53b7c5fe23ca2c3e4c49, ASSIGN because future has completed 2024-11-24T19:56:33,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 566da3ef551e53b7c5fe23ca2c3e4c49, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:56:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-24T19:56:33,661 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:56:33,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 566da3ef551e53b7c5fe23ca2c3e4c49, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:56:33,662 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,662 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:33,663 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,663 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,666 INFO [StoreOpener-566da3ef551e53b7c5fe23ca2c3e4c49-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,669 INFO [StoreOpener-566da3ef551e53b7c5fe23ca2c3e4c49-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 566da3ef551e53b7c5fe23ca2c3e4c49 columnFamilyName testFamily 2024-11-24T19:56:33,669 DEBUG [StoreOpener-566da3ef551e53b7c5fe23ca2c3e4c49-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:33,669 INFO [StoreOpener-566da3ef551e53b7c5fe23ca2c3e4c49-1 {}] regionserver.HStore(327): Store=566da3ef551e53b7c5fe23ca2c3e4c49/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:56:33,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,671 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,672 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,672 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,672 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,679 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:56:33,680 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 566da3ef551e53b7c5fe23ca2c3e4c49; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59422262, jitterRate=-0.11453929543495178}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:56:33,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:56:33,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 566da3ef551e53b7c5fe23ca2c3e4c49: Running coprocessor pre-open hook at 1732478193663Writing region info on filesystem at 1732478193663Initializing all the Stores at 1732478193665 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478193665Cleaning up temporary data from old regions at 1732478193673 (+8 ms)Running coprocessor post-open hooks at 1732478193680 (+7 ms)Region opened successfully at 1732478193680 2024-11-24T19:56:33,682 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49., pid=12, masterSystemTime=1732478193651 2024-11-24T19:56:33,684 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:56:33,684 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:56:33,685 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=566da3ef551e53b7c5fe23ca2c3e4c49, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:33,688 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 566da3ef551e53b7c5fe23ca2c3e4c49, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:56:33,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T19:56:33,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 566da3ef551e53b7c5fe23ca2c3e4c49, server=32a03c886245,38229,1732478146827 in 193 msec 2024-11-24T19:56:33,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T19:56:33,697 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=566da3ef551e53b7c5fe23ca2c3e4c49, ASSIGN in 363 msec 2024-11-24T19:56:33,698 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:56:33,698 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478193698"}]},"ts":"1732478193698"} 2024-11-24T19:56:33,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-24T19:56:33,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:56:33,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2270 sec 2024-11-24T19:56:35,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-24T19:56:35,521 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-24T19:56:35,521 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-24T19:56:35,522 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:35,532 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-24T19:56:35,532 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:35,533 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-24T19:56:35,536 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:56:35,546 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:56:35,546 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-24T19:56:35,549 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-24T19:56:35,551 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-24T19:56:35,553 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-24T19:56:35,553 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-24T19:56:36,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-24T19:56:36,451 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T19:56:37,344 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-24T19:56:39,194 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:56:41,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38229 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027927019' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T19:56:41,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38229 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:40398 deadline: 1732478207571, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027927019' does not exist 2024-11-24T19:56:41,575 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 32a03c886245,38229,1732478146827 for closing scanner id = -3309366211027927019 for 566da3ef551e53b7c5fe23ca2c3e4c49 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027927019' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T19:56:41,578 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=158 (was 171), ProcessCount=11 (was 11), AvailableMemoryMB=10736 (was 10744) 2024-11-24T19:56:41,587 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=158, ProcessCount=11, AvailableMemoryMB=10735 2024-11-24T19:56:41,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:56:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-24T19:56:41,592 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:56:41,592 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:41,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-24T19:56:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T19:56:41,594 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:56:41,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741841_1017 (size=423) 2024-11-24T19:56:42,004 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7f8a6d58dc9d29ed781574e1b58a0094, NAME => 'testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:56:42,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741842_1018 (size=74) 2024-11-24T19:56:42,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-24T19:56:42,414 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:42,414 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing 7f8a6d58dc9d29ed781574e1b58a0094, disabling compactions & flushes 2024-11-24T19:56:42,414 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:56:42,414 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:56:42,414 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. after waiting 0 ms 2024-11-24T19:56:42,414 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:56:42,414 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:56:42,415 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7f8a6d58dc9d29ed781574e1b58a0094: Waiting for close lock at 1732478202414Disabling compacts and flushes for region at 1732478202414Disabling writes for close at 1732478202414Writing region close event to WAL at 1732478202414Closed at 1732478202414 2024-11-24T19:56:42,418 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:56:42,419 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732478202418"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478202418"}]},"ts":"1732478202418"} 2024-11-24T19:56:42,423 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:56:42,425 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:56:42,425 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478202425"}]},"ts":"1732478202425"} 2024-11-24T19:56:42,428 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-24T19:56:42,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7f8a6d58dc9d29ed781574e1b58a0094, ASSIGN}] 2024-11-24T19:56:42,430 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7f8a6d58dc9d29ed781574e1b58a0094, ASSIGN 2024-11-24T19:56:42,432 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7f8a6d58dc9d29ed781574e1b58a0094, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:56:42,583 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=7f8a6d58dc9d29ed781574e1b58a0094, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:42,593 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7f8a6d58dc9d29ed781574e1b58a0094, ASSIGN because future has completed 2024-11-24T19:56:42,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f8a6d58dc9d29ed781574e1b58a0094, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:56:42,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T19:56:42,759 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:56:42,759 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => 7f8a6d58dc9d29ed781574e1b58a0094, NAME => 'testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:56:42,760 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,760 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:42,760 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,760 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,763 INFO [StoreOpener-7f8a6d58dc9d29ed781574e1b58a0094-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,768 INFO [StoreOpener-7f8a6d58dc9d29ed781574e1b58a0094-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7f8a6d58dc9d29ed781574e1b58a0094 columnFamilyName testFamily 2024-11-24T19:56:42,768 DEBUG [StoreOpener-7f8a6d58dc9d29ed781574e1b58a0094-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:42,769 INFO [StoreOpener-7f8a6d58dc9d29ed781574e1b58a0094-1 {}] regionserver.HStore(327): Store=7f8a6d58dc9d29ed781574e1b58a0094/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:56:42,769 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,771 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,772 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,772 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,772 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,775 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,778 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:56:42,779 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened 7f8a6d58dc9d29ed781574e1b58a0094; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62273772, jitterRate=-0.07204848527908325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:56:42,779 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:56:42,780 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for 7f8a6d58dc9d29ed781574e1b58a0094: Running coprocessor pre-open hook at 1732478202760Writing region info on filesystem at 1732478202760Initializing all the Stores at 1732478202763 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478202763Cleaning up temporary data from old regions at 1732478202772 (+9 ms)Running coprocessor post-open hooks at 1732478202779 (+7 ms)Region opened successfully at 1732478202780 (+1 ms) 2024-11-24T19:56:42,781 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094., pid=15, masterSystemTime=1732478202748 2024-11-24T19:56:42,784 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:56:42,784 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:56:42,785 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=7f8a6d58dc9d29ed781574e1b58a0094, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:42,788 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7f8a6d58dc9d29ed781574e1b58a0094, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:56:42,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T19:56:42,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 7f8a6d58dc9d29ed781574e1b58a0094, server=32a03c886245,38229,1732478146827 in 195 msec 2024-11-24T19:56:42,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T19:56:42,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=7f8a6d58dc9d29ed781574e1b58a0094, ASSIGN in 364 msec 2024-11-24T19:56:42,797 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:56:42,797 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478202797"}]},"ts":"1732478202797"} 2024-11-24T19:56:42,800 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-24T19:56:42,801 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:56:42,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2120 sec 2024-11-24T19:56:43,347 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-24T19:56:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T19:56:44,630 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-24T19:56:44,631 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-24T19:56:44,631 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:44,640 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-24T19:56:44,641 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:44,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-24T19:56:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:56:44,655 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:56:44,655 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-24T19:56:44,658 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-24T19:56:44,661 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-24T19:56:44,662 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-24T19:56:44,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3309366211027927015 2024-11-24T19:56:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40398 deadline: 1732478207661, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T19:56:44,667 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-24T19:56:44,668 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-24T19:56:44,669 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:44,671 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:44,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-3309366211027927013 2024-11-24T19:56:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40398 deadline: 1732478207670, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T19:56:44,675 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:44,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-3309366211027927012 2024-11-24T19:56:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40398 deadline: 1732478207674, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T19:56:44,679 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:44,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-3309366211027927011 2024-11-24T19:56:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:40398 deadline: 1732478207678, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-24T19:56:44,683 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-24T19:56:44,692 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=226 (was 225) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1545303662_22 at /127.0.0.1:45930 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=145 (was 158), ProcessCount=11 (was 11), AvailableMemoryMB=10720 (was 10735) 2024-11-24T19:56:44,699 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=226, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=145, ProcessCount=11, AvailableMemoryMB=10720 2024-11-24T19:56:44,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:56:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-24T19:56:44,704 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:56:44,704 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:44,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-24T19:56:44,706 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:56:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-24T19:56:44,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741843_1019 (size=418) 2024-11-24T19:56:45,122 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8b52240251203d3232be1c4491b831e4, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:56:45,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741844_1020 (size=69) 2024-11-24T19:56:45,268 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T19:56:45,533 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:45,533 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 8b52240251203d3232be1c4491b831e4, disabling compactions & flushes 2024-11-24T19:56:45,534 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:56:45,534 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:56:45,534 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. after waiting 0 ms 2024-11-24T19:56:45,534 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:56:45,534 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:56:45,534 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8b52240251203d3232be1c4491b831e4: Waiting for close lock at 1732478205533Disabling compacts and flushes for region at 1732478205533Disabling writes for close at 1732478205534 (+1 ms)Writing region close event to WAL at 1732478205534Closed at 1732478205534 2024-11-24T19:56:45,538 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:56:45,539 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732478205538"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478205538"}]},"ts":"1732478205538"} 2024-11-24T19:56:45,543 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:56:45,545 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:56:45,545 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478205545"}]},"ts":"1732478205545"} 2024-11-24T19:56:45,547 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-24T19:56:45,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8b52240251203d3232be1c4491b831e4, ASSIGN}] 2024-11-24T19:56:45,549 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8b52240251203d3232be1c4491b831e4, ASSIGN 2024-11-24T19:56:45,551 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8b52240251203d3232be1c4491b831e4, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:56:45,702 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=8b52240251203d3232be1c4491b831e4, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:45,712 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8b52240251203d3232be1c4491b831e4, ASSIGN because future has completed 2024-11-24T19:56:45,713 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b52240251203d3232be1c4491b831e4, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:56:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-24T19:56:45,876 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:56:45,876 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b52240251203d3232be1c4491b831e4, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:56:45,877 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,877 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:45,878 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,878 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,880 INFO [StoreOpener-8b52240251203d3232be1c4491b831e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,882 INFO [StoreOpener-8b52240251203d3232be1c4491b831e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b52240251203d3232be1c4491b831e4 columnFamilyName testFamily 2024-11-24T19:56:45,882 DEBUG [StoreOpener-8b52240251203d3232be1c4491b831e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:45,882 INFO [StoreOpener-8b52240251203d3232be1c4491b831e4-1 {}] regionserver.HStore(327): Store=8b52240251203d3232be1c4491b831e4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:56:45,883 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,884 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,885 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,885 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,885 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,887 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,890 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:56:45,891 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 8b52240251203d3232be1c4491b831e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68097786, jitterRate=0.014736086130142212}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:56:45,891 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b52240251203d3232be1c4491b831e4 2024-11-24T19:56:45,892 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 8b52240251203d3232be1c4491b831e4: Running coprocessor pre-open hook at 1732478205878Writing region info on filesystem at 1732478205878Initializing all the Stores at 1732478205879 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478205880 (+1 ms)Cleaning up temporary data from old regions at 1732478205885 (+5 ms)Running coprocessor post-open hooks at 1732478205891 (+6 ms)Region opened successfully at 1732478205892 (+1 ms) 2024-11-24T19:56:45,893 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., pid=18, masterSystemTime=1732478205866 2024-11-24T19:56:45,896 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:56:45,896 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:56:45,897 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=8b52240251203d3232be1c4491b831e4, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:45,899 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b52240251203d3232be1c4491b831e4, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:56:45,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T19:56:45,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b52240251203d3232be1c4491b831e4, server=32a03c886245,38229,1732478146827 in 189 msec 2024-11-24T19:56:45,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T19:56:45,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=8b52240251203d3232be1c4491b831e4, ASSIGN in 357 msec 2024-11-24T19:56:45,910 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:56:45,910 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478205910"}]},"ts":"1732478205910"} 2024-11-24T19:56:45,913 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-24T19:56:45,914 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:56:45,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2130 sec 2024-11-24T19:56:46,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-24T19:56:46,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-24T19:56:46,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-24T19:56:46,454 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-24T19:56:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-24T19:56:47,738 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-24T19:56:47,738 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-24T19:56:47,738 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:47,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-24T19:56:47,743 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:47,743 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-24T19:56:47,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:56:47,761 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:56:47,761 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-24T19:56:47,765 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-24T19:56:47,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-24T19:56:48,264 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:56:48,772 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=32a03c886245:38229 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-24T19:56:48,773 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1006ms, rpcTimeout=1000ms 2024-11-24T19:56:48,773 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2 from cache 2024-11-24T19:56:49,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-24T19:56:49,350 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-24T19:56:49,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:56:49,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-24T19:56:50,772 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=32a03c886245:38229 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=975ms, rpcTimeout=970ms 2024-11-24T19:56:50,772 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=975ms, rpcTimeout=970ms 2024-11-24T19:56:50,772 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., hostname=32a03c886245,38229,1732478146827, seqNum=2 from cache 2024-11-24T19:56:50,790 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 226), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=133 (was 145), ProcessCount=11 (was 11), AvailableMemoryMB=10713 (was 10720) 2024-11-24T19:56:50,798 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=133, ProcessCount=11, AvailableMemoryMB=10713 2024-11-24T19:56:50,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:56:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-24T19:56:50,805 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:56:50,805 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:50,805 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-24T19:56:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T19:56:50,807 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:56:50,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741845_1021 (size=416) 2024-11-24T19:56:51,223 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 65a3df80a12c37fb9fbab62e0bca8061, NAME => 'testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:56:51,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741846_1022 (size=67) 2024-11-24T19:56:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-24T19:56:51,636 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:51,636 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 65a3df80a12c37fb9fbab62e0bca8061, disabling compactions & flushes 2024-11-24T19:56:51,636 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:56:51,636 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:56:51,637 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. after waiting 0 ms 2024-11-24T19:56:51,637 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:56:51,637 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:56:51,638 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 65a3df80a12c37fb9fbab62e0bca8061: Waiting for close lock at 1732478211636Disabling compacts and flushes for region at 1732478211636Disabling writes for close at 1732478211637 (+1 ms)Writing region close event to WAL at 1732478211637Closed at 1732478211637 2024-11-24T19:56:51,641 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:56:51,641 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732478211641"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478211641"}]},"ts":"1732478211641"} 2024-11-24T19:56:51,645 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:56:51,647 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:56:51,647 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478211647"}]},"ts":"1732478211647"} 2024-11-24T19:56:51,651 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-24T19:56:51,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=65a3df80a12c37fb9fbab62e0bca8061, ASSIGN}] 2024-11-24T19:56:51,653 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=65a3df80a12c37fb9fbab62e0bca8061, ASSIGN 2024-11-24T19:56:51,655 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=65a3df80a12c37fb9fbab62e0bca8061, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:56:51,807 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=65a3df80a12c37fb9fbab62e0bca8061, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:51,815 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=65a3df80a12c37fb9fbab62e0bca8061, ASSIGN because future has completed 2024-11-24T19:56:51,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65a3df80a12c37fb9fbab62e0bca8061, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:56:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T19:56:51,987 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:56:51,987 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 65a3df80a12c37fb9fbab62e0bca8061, NAME => 'testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:56:51,988 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,988 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:51,988 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,988 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,990 INFO [StoreOpener-65a3df80a12c37fb9fbab62e0bca8061-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,992 INFO [StoreOpener-65a3df80a12c37fb9fbab62e0bca8061-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65a3df80a12c37fb9fbab62e0bca8061 columnFamilyName testFamily 2024-11-24T19:56:51,992 DEBUG [StoreOpener-65a3df80a12c37fb9fbab62e0bca8061-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:51,993 INFO [StoreOpener-65a3df80a12c37fb9fbab62e0bca8061-1 {}] regionserver.HStore(327): Store=65a3df80a12c37fb9fbab62e0bca8061/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:56:51,993 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,994 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,994 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,995 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,995 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:51,997 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:52,000 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:56:52,001 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 65a3df80a12c37fb9fbab62e0bca8061; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60995546, jitterRate=-0.09109553694725037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:56:52,001 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:56:52,002 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 65a3df80a12c37fb9fbab62e0bca8061: Running coprocessor pre-open hook at 1732478211989Writing region info on filesystem at 1732478211989Initializing all the Stores at 1732478211990 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478211990Cleaning up temporary data from old regions at 1732478211995 (+5 ms)Running coprocessor post-open hooks at 1732478212001 (+6 ms)Region opened successfully at 1732478212002 (+1 ms) 2024-11-24T19:56:52,003 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061., pid=21, masterSystemTime=1732478211973 2024-11-24T19:56:52,006 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:56:52,007 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:56:52,007 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=65a3df80a12c37fb9fbab62e0bca8061, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:52,011 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 65a3df80a12c37fb9fbab62e0bca8061, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:56:52,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-24T19:56:52,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 65a3df80a12c37fb9fbab62e0bca8061, server=32a03c886245,38229,1732478146827 in 197 msec 2024-11-24T19:56:52,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T19:56:52,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=65a3df80a12c37fb9fbab62e0bca8061, ASSIGN in 364 msec 2024-11-24T19:56:52,019 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:56:52,020 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478212019"}]},"ts":"1732478212019"} 2024-11-24T19:56:52,022 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-24T19:56:52,023 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:56:52,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2220 sec 2024-11-24T19:56:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T19:56:53,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-24T19:56:53,831 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-24T19:56:53,832 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:53,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-24T19:56:53,843 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:53,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-24T19:56:53,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:56:53,858 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:56:53,858 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-24T19:56:53,862 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-24T19:56:53,864 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-24T19:56:53,865 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-24T19:56:53,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-24T19:56:55,352 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-24T19:56:56,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-24T19:56:56,451 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T19:56:56,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38229 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027927000' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T19:56:56,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38229 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:40398 deadline: 1732478217875, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027927000' does not exist 2024-11-24T19:56:56,876 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 32a03c886245,38229,1732478146827 for closing scanner id = -3309366211027927000 for 65a3df80a12c37fb9fbab62e0bca8061 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027927000' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T19:56:56,880 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=122 (was 133), ProcessCount=11 (was 11), AvailableMemoryMB=10715 (was 10713) - AvailableMemoryMB LEAK? - 2024-11-24T19:56:56,888 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=11, AvailableMemoryMB=10715 2024-11-24T19:56:56,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:56:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-24T19:56:56,893 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:56:56,893 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:56,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-24T19:56:56,895 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:56:56,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-24T19:56:56,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741847_1023 (size=411) 2024-11-24T19:56:57,307 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 184906bf8a6e152a8c22472839b543e1, NAME => 'testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:56:57,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741848_1024 (size=62) 2024-11-24T19:56:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-24T19:56:57,492 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:56:57,722 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:57,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 184906bf8a6e152a8c22472839b543e1, disabling compactions & flushes 2024-11-24T19:56:57,723 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:56:57,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:56:57,723 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. after waiting 0 ms 2024-11-24T19:56:57,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:56:57,724 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:56:57,724 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 184906bf8a6e152a8c22472839b543e1: Waiting for close lock at 1732478217723Disabling compacts and flushes for region at 1732478217723Disabling writes for close at 1732478217723Writing region close event to WAL at 1732478217724 (+1 ms)Closed at 1732478217724 2024-11-24T19:56:57,727 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:56:57,728 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732478217727"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478217727"}]},"ts":"1732478217727"} 2024-11-24T19:56:57,730 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:56:57,732 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:56:57,732 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478217732"}]},"ts":"1732478217732"} 2024-11-24T19:56:57,735 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-24T19:56:57,735 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=184906bf8a6e152a8c22472839b543e1, ASSIGN}] 2024-11-24T19:56:57,737 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=184906bf8a6e152a8c22472839b543e1, ASSIGN 2024-11-24T19:56:57,738 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=184906bf8a6e152a8c22472839b543e1, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:56:57,890 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=184906bf8a6e152a8c22472839b543e1, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:57,897 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=184906bf8a6e152a8c22472839b543e1, ASSIGN because future has completed 2024-11-24T19:56:57,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 184906bf8a6e152a8c22472839b543e1, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:56:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-24T19:56:58,066 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:56:58,067 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 184906bf8a6e152a8c22472839b543e1, NAME => 'testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:56:58,067 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,067 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:56:58,068 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,068 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,071 INFO [StoreOpener-184906bf8a6e152a8c22472839b543e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,073 INFO [StoreOpener-184906bf8a6e152a8c22472839b543e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 184906bf8a6e152a8c22472839b543e1 columnFamilyName testFamily 2024-11-24T19:56:58,074 DEBUG [StoreOpener-184906bf8a6e152a8c22472839b543e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:56:58,075 INFO [StoreOpener-184906bf8a6e152a8c22472839b543e1-1 {}] regionserver.HStore(327): Store=184906bf8a6e152a8c22472839b543e1/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:56:58,075 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,076 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,077 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,078 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,078 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,081 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,085 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:56:58,086 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 184906bf8a6e152a8c22472839b543e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62395781, jitterRate=-0.0702304095029831}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:56:58,086 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:56:58,086 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 184906bf8a6e152a8c22472839b543e1: Running coprocessor pre-open hook at 1732478218068Writing region info on filesystem at 1732478218068Initializing all the Stores at 1732478218070 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478218070Cleaning up temporary data from old regions at 1732478218078 (+8 ms)Running coprocessor post-open hooks at 1732478218086 (+8 ms)Region opened successfully at 1732478218086 2024-11-24T19:56:58,087 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1., pid=24, masterSystemTime=1732478218053 2024-11-24T19:56:58,090 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:56:58,090 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:56:58,090 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=184906bf8a6e152a8c22472839b543e1, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:56:58,093 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 184906bf8a6e152a8c22472839b543e1, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:56:58,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T19:56:58,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 184906bf8a6e152a8c22472839b543e1, server=32a03c886245,38229,1732478146827 in 196 msec 2024-11-24T19:56:58,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-24T19:56:58,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=184906bf8a6e152a8c22472839b543e1, ASSIGN in 362 msec 2024-11-24T19:56:58,101 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:56:58,101 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478218101"}]},"ts":"1732478218101"} 2024-11-24T19:56:58,104 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-24T19:56:58,105 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:56:58,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2150 sec 2024-11-24T19:56:59,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-24T19:56:59,939 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-24T19:56:59,939 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-24T19:56:59,940 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:59,948 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-24T19:56:59,949 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:56:59,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-24T19:56:59,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:56:59,965 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:56:59,965 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-24T19:56:59,969 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-24T19:56:59,971 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-24T19:56:59,973 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-24T19:56:59,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-24T19:57:00,640 INFO [master/32a03c886245:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T19:57:00,640 INFO [master/32a03c886245:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T19:57:01,355 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-24T19:57:02,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38229 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027926996' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T19:57:02,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38229 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:40398 deadline: 1732478223989, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027926996' does not exist 2024-11-24T19:57:02,993 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 32a03c886245,38229,1732478146827 for closing scanner id = -3309366211027926996 for 184906bf8a6e152a8c22472839b543e1 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027926996' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T19:57:03,000 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=103 (was 122), ProcessCount=11 (was 11), AvailableMemoryMB=10726 (was 10715) - AvailableMemoryMB LEAK? - 2024-11-24T19:57:03,008 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=103, ProcessCount=11, AvailableMemoryMB=10726 2024-11-24T19:57:03,010 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:57:03,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-24T19:57:03,014 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:57:03,014 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:57:03,014 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-24T19:57:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T19:57:03,016 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:57:03,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741849_1025 (size=419) 2024-11-24T19:57:03,432 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3ca11ada5ddb9634563081985fee5db9, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:57:03,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741850_1026 (size=70) 2024-11-24T19:57:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-24T19:57:03,571 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:57:03,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:57:03,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 3ca11ada5ddb9634563081985fee5db9, disabling compactions & flushes 2024-11-24T19:57:03,847 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:03,847 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:03,848 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. after waiting 0 ms 2024-11-24T19:57:03,848 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:03,848 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:03,848 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3ca11ada5ddb9634563081985fee5db9: Waiting for close lock at 1732478223847Disabling compacts and flushes for region at 1732478223847Disabling writes for close at 1732478223848 (+1 ms)Writing region close event to WAL at 1732478223848Closed at 1732478223848 2024-11-24T19:57:03,852 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:57:03,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732478223852"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478223852"}]},"ts":"1732478223852"} 2024-11-24T19:57:03,855 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:57:03,856 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:57:03,857 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478223856"}]},"ts":"1732478223856"} 2024-11-24T19:57:03,859 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-24T19:57:03,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3ca11ada5ddb9634563081985fee5db9, ASSIGN}] 2024-11-24T19:57:03,862 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3ca11ada5ddb9634563081985fee5db9, ASSIGN 2024-11-24T19:57:03,863 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3ca11ada5ddb9634563081985fee5db9, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:57:04,013 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=3ca11ada5ddb9634563081985fee5db9, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:57:04,017 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3ca11ada5ddb9634563081985fee5db9, ASSIGN because future has completed 2024-11-24T19:57:04,017 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ca11ada5ddb9634563081985fee5db9, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:57:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T19:57:04,183 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:04,183 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ca11ada5ddb9634563081985fee5db9, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:57:04,184 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,184 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:57:04,184 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,184 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,187 INFO [StoreOpener-3ca11ada5ddb9634563081985fee5db9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,190 INFO [StoreOpener-3ca11ada5ddb9634563081985fee5db9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ca11ada5ddb9634563081985fee5db9 columnFamilyName testFamily 2024-11-24T19:57:04,190 DEBUG [StoreOpener-3ca11ada5ddb9634563081985fee5db9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:57:04,191 INFO [StoreOpener-3ca11ada5ddb9634563081985fee5db9-1 {}] regionserver.HStore(327): Store=3ca11ada5ddb9634563081985fee5db9/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:57:04,191 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,192 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,194 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,197 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,200 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:57:04,200 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 3ca11ada5ddb9634563081985fee5db9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71484213, jitterRate=0.06519778072834015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:57:04,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:04,201 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 3ca11ada5ddb9634563081985fee5db9: Running coprocessor pre-open hook at 1732478224184Writing region info on filesystem at 1732478224184Initializing all the Stores at 1732478224187 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478224187Cleaning up temporary data from old regions at 1732478224194 (+7 ms)Running coprocessor post-open hooks at 1732478224201 (+7 ms)Region opened successfully at 1732478224201 2024-11-24T19:57:04,202 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., pid=27, masterSystemTime=1732478224170 2024-11-24T19:57:04,205 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:04,205 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:04,206 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=3ca11ada5ddb9634563081985fee5db9, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:57:04,208 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ca11ada5ddb9634563081985fee5db9, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:57:04,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-24T19:57:04,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ca11ada5ddb9634563081985fee5db9, server=32a03c886245,38229,1732478146827 in 192 msec 2024-11-24T19:57:04,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T19:57:04,215 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=3ca11ada5ddb9634563081985fee5db9, ASSIGN in 352 msec 2024-11-24T19:57:04,215 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:57:04,216 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478224215"}]},"ts":"1732478224215"} 2024-11-24T19:57:04,218 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-24T19:57:04,219 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:57:04,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2090 sec 2024-11-24T19:57:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T19:57:06,051 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-24T19:57:06,051 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-24T19:57:06,052 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:57:06,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-24T19:57:06,063 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:57:06,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-24T19:57:06,067 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:57:06,076 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:57:06,076 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-24T19:57:06,079 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-24T19:57:06,081 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-24T19:57:06,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-24T19:57:06,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-24T19:57:06,454 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-24T19:57:06,454 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T19:57:07,358 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-24T19:57:09,690 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:57:12,082 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., hostname=32a03c886245,38229,1732478146827, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., hostname=32a03c886245,38229,1732478146827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=32a03c886245:38229 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-11-24T19:57:12,082 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., hostname=32a03c886245,38229,1732478146827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6001ms, rpcTimeout=5999ms 2024-11-24T19:57:12,082 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., hostname=32a03c886245,38229,1732478146827, seqNum=2 from cache 2024-11-24T19:57:12,098 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=95 (was 103), ProcessCount=11 (was 11), AvailableMemoryMB=10729 (was 10726) - AvailableMemoryMB LEAK? - 2024-11-24T19:57:12,107 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=95, ProcessCount=11, AvailableMemoryMB=10729 2024-11-24T19:57:12,109 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:57:12,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-24T19:57:12,111 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:57:12,111 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:57:12,112 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-24T19:57:12,113 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:57:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-24T19:57:12,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741851_1027 (size=407) 2024-11-24T19:57:12,528 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eede8119663a58fe5707f1b4ce9249e4, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:57:12,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741852_1028 (size=58) 2024-11-24T19:57:12,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-24T19:57:12,941 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:57:12,941 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing eede8119663a58fe5707f1b4ce9249e4, disabling compactions & flushes 2024-11-24T19:57:12,941 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:12,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:12,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. after waiting 0 ms 2024-11-24T19:57:12,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:12,942 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:12,942 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for eede8119663a58fe5707f1b4ce9249e4: Waiting for close lock at 1732478232941Disabling compacts and flushes for region at 1732478232941Disabling writes for close at 1732478232942 (+1 ms)Writing region close event to WAL at 1732478232942Closed at 1732478232942 2024-11-24T19:57:12,947 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:57:12,948 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732478232947"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478232947"}]},"ts":"1732478232947"} 2024-11-24T19:57:12,951 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:57:12,952 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:57:12,953 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478232952"}]},"ts":"1732478232952"} 2024-11-24T19:57:12,955 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-24T19:57:12,955 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eede8119663a58fe5707f1b4ce9249e4, ASSIGN}] 2024-11-24T19:57:12,957 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eede8119663a58fe5707f1b4ce9249e4, ASSIGN 2024-11-24T19:57:12,958 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eede8119663a58fe5707f1b4ce9249e4, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:57:13,109 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=eede8119663a58fe5707f1b4ce9249e4, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:57:13,115 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eede8119663a58fe5707f1b4ce9249e4, ASSIGN because future has completed 2024-11-24T19:57:13,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure eede8119663a58fe5707f1b4ce9249e4, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:57:13,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-24T19:57:13,279 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:13,279 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => eede8119663a58fe5707f1b4ce9249e4, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:57:13,280 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,280 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:57:13,281 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,281 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,284 INFO [StoreOpener-eede8119663a58fe5707f1b4ce9249e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,287 INFO [StoreOpener-eede8119663a58fe5707f1b4ce9249e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eede8119663a58fe5707f1b4ce9249e4 columnFamilyName testFamily 2024-11-24T19:57:13,287 DEBUG [StoreOpener-eede8119663a58fe5707f1b4ce9249e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:57:13,288 INFO [StoreOpener-eede8119663a58fe5707f1b4ce9249e4-1 {}] regionserver.HStore(327): Store=eede8119663a58fe5707f1b4ce9249e4/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:57:13,288 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,289 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,289 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,290 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,292 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,295 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:57:13,296 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened eede8119663a58fe5707f1b4ce9249e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65678891, jitterRate=-0.021308258175849915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:57:13,296 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:13,297 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for eede8119663a58fe5707f1b4ce9249e4: Running coprocessor pre-open hook at 1732478233281Writing region info on filesystem at 1732478233281Initializing all the Stores at 1732478233283 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478233284 (+1 ms)Cleaning up temporary data from old regions at 1732478233290 (+6 ms)Running coprocessor post-open hooks at 1732478233296 (+6 ms)Region opened successfully at 1732478233297 (+1 ms) 2024-11-24T19:57:13,298 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4., pid=30, masterSystemTime=1732478233270 2024-11-24T19:57:13,301 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:13,301 INFO [RS_OPEN_PRIORITY_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:13,301 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=eede8119663a58fe5707f1b4ce9249e4, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:57:13,304 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure eede8119663a58fe5707f1b4ce9249e4, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:57:13,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-24T19:57:13,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure eede8119663a58fe5707f1b4ce9249e4, server=32a03c886245,38229,1732478146827 in 189 msec 2024-11-24T19:57:13,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-24T19:57:13,312 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=eede8119663a58fe5707f1b4ce9249e4, ASSIGN in 354 msec 2024-11-24T19:57:13,313 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:57:13,313 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478233313"}]},"ts":"1732478233313"} 2024-11-24T19:57:13,316 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-24T19:57:13,317 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:57:13,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2080 sec 2024-11-24T19:57:13,361 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-24T19:57:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-24T19:57:15,129 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-24T19:57:15,129 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-24T19:57:15,129 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:57:15,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-24T19:57:15,138 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:57:15,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-24T19:57:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:57:15,150 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:57:15,150 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-24T19:57:15,153 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-24T19:57:15,155 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-24T19:57:15,156 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-24T19:57:15,157 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-24T19:57:15,268 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T19:57:16,452 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-24T19:57:16,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-24T19:57:18,786 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:57:19,984 DEBUG [master/32a03c886245:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T19:57:21,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38229 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027926988' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-24T19:57:21,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38229 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:40398 deadline: 1732478247168, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027926988' does not exist 2024-11-24T19:57:21,170 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 32a03c886245,38229,1732478146827 for closing scanner id = -3309366211027926988 for eede8119663a58fe5707f1b4ce9249e4 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-3309366211027926988' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T19:57:21,176 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=80 (was 95), ProcessCount=11 (was 11), AvailableMemoryMB=10722 (was 10729) 2024-11-24T19:57:21,184 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=80, ProcessCount=11, AvailableMemoryMB=10722 2024-11-24T19:57:21,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T19:57:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-24T19:57:21,189 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T19:57:21,189 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:57:21,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-24T19:57:21,190 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T19:57:21,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-24T19:57:21,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741853_1029 (size=423) 2024-11-24T19:57:21,202 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 024525ffa8c3c04085ca40ac2c8fc04e, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f 2024-11-24T19:57:21,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741854_1030 (size=74) 2024-11-24T19:57:21,610 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:57:21,611 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 024525ffa8c3c04085ca40ac2c8fc04e, disabling compactions & flushes 2024-11-24T19:57:21,611 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:21,611 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:21,611 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. after waiting 0 ms 2024-11-24T19:57:21,611 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:21,611 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:21,611 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 024525ffa8c3c04085ca40ac2c8fc04e: Waiting for close lock at 1732478241610Disabling compacts and flushes for region at 1732478241611 (+1 ms)Disabling writes for close at 1732478241611Writing region close event to WAL at 1732478241611Closed at 1732478241611 2024-11-24T19:57:21,613 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T19:57:21,613 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732478241613"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732478241613"}]},"ts":"1732478241613"} 2024-11-24T19:57:21,617 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T19:57:21,619 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T19:57:21,619 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478241619"}]},"ts":"1732478241619"} 2024-11-24T19:57:21,622 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-24T19:57:21,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=024525ffa8c3c04085ca40ac2c8fc04e, ASSIGN}] 2024-11-24T19:57:21,624 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=024525ffa8c3c04085ca40ac2c8fc04e, ASSIGN 2024-11-24T19:57:21,626 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=024525ffa8c3c04085ca40ac2c8fc04e, ASSIGN; state=OFFLINE, location=32a03c886245,38229,1732478146827; forceNewPlan=false, retain=false 2024-11-24T19:57:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-24T19:57:21,777 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=024525ffa8c3c04085ca40ac2c8fc04e, regionState=OPENING, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:57:21,783 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=024525ffa8c3c04085ca40ac2c8fc04e, ASSIGN because future has completed 2024-11-24T19:57:21,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 024525ffa8c3c04085ca40ac2c8fc04e, server=32a03c886245,38229,1732478146827}] 2024-11-24T19:57:21,946 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:21,947 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 024525ffa8c3c04085ca40ac2c8fc04e, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e.', STARTKEY => '', ENDKEY => ''} 2024-11-24T19:57:21,947 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,947 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T19:57:21,947 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,947 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,950 INFO [StoreOpener-024525ffa8c3c04085ca40ac2c8fc04e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,952 INFO [StoreOpener-024525ffa8c3c04085ca40ac2c8fc04e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 024525ffa8c3c04085ca40ac2c8fc04e columnFamilyName testFamily 2024-11-24T19:57:21,952 DEBUG [StoreOpener-024525ffa8c3c04085ca40ac2c8fc04e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T19:57:21,953 INFO [StoreOpener-024525ffa8c3c04085ca40ac2c8fc04e-1 {}] regionserver.HStore(327): Store=024525ffa8c3c04085ca40ac2c8fc04e/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T19:57:21,953 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,954 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,954 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,955 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,955 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,957 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,960 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T19:57:21,960 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 024525ffa8c3c04085ca40ac2c8fc04e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67859199, jitterRate=0.011180862784385681}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T19:57:21,961 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:21,961 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 024525ffa8c3c04085ca40ac2c8fc04e: Running coprocessor pre-open hook at 1732478241948Writing region info on filesystem at 1732478241948Initializing all the Stores at 1732478241950 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732478241950Cleaning up temporary data from old regions at 1732478241955 (+5 ms)Running coprocessor post-open hooks at 1732478241961 (+6 ms)Region opened successfully at 1732478241961 2024-11-24T19:57:21,962 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., pid=33, masterSystemTime=1732478241937 2024-11-24T19:57:21,965 DEBUG [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:21,965 INFO [RS_OPEN_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:21,966 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=024525ffa8c3c04085ca40ac2c8fc04e, regionState=OPEN, openSeqNum=2, regionLocation=32a03c886245,38229,1732478146827 2024-11-24T19:57:21,969 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 024525ffa8c3c04085ca40ac2c8fc04e, server=32a03c886245,38229,1732478146827 because future has completed 2024-11-24T19:57:21,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-24T19:57:21,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 024525ffa8c3c04085ca40ac2c8fc04e, server=32a03c886245,38229,1732478146827 in 187 msec 2024-11-24T19:57:21,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-24T19:57:21,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=024525ffa8c3c04085ca40ac2c8fc04e, ASSIGN in 352 msec 2024-11-24T19:57:21,977 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T19:57:21,977 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732478241977"}]},"ts":"1732478241977"} 2024-11-24T19:57:21,979 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-24T19:57:21,980 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T19:57:21,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 795 msec 2024-11-24T19:57:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45559 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-24T19:57:22,201 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-24T19:57:22,202 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-24T19:57:22,203 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:57:22,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-24T19:57:22,213 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T19:57:22,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-24T19:57:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:57:22,227 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-24T19:57:22,228 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-24T19:57:22,231 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-24T19:57:22,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-24T19:57:23,243 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=32a03c886245:38229 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1011ms, rpcTimeout=1000ms 2024-11-24T19:57:23,243 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1011ms, rpcTimeout=1000ms 2024-11-24T19:57:23,243 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2 from cache 2024-11-24T19:57:23,733 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-24T19:57:24,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2] 2024-11-24T19:57:24,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38229 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-24T19:57:25,242 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=32a03c886245:38229 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=976ms, rpcTimeout=965ms 2024-11-24T19:57:25,242 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=976ms, rpcTimeout=965ms 2024-11-24T19:57:25,242 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., hostname=32a03c886245,38229,1732478146827, seqNum=2 from cache 2024-11-24T19:57:25,249 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=224 (was 223) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1545303662_22 at /127.0.0.1:39584 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=74 (was 80), ProcessCount=11 (was 11), AvailableMemoryMB=10721 (was 10722) 2024-11-24T19:57:25,250 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T19:57:25,250 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:57:25,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:57:25,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:57:25,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T19:57:25,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T19:57:25,255 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T19:57:25,255 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:57:25,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:57:25,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:57:25,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T19:57:25,256 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T19:57:25,256 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1780335685, stopped=false 2024-11-24T19:57:25,256 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=32a03c886245,45559,1732478146124 2024-11-24T19:57:25,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T19:57:25,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T19:57:25,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:57:25,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:57:25,301 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T19:57:25,302 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T19:57:25,302 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:57:25,302 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:57:25,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:57:25,302 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T19:57:25,303 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '32a03c886245,38229,1732478146827' ***** 2024-11-24T19:57:25,303 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T19:57:25,304 INFO [RS:0;32a03c886245:38229 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T19:57:25,304 INFO [RS:0;32a03c886245:38229 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T19:57:25,304 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T19:57:25,304 INFO [RS:0;32a03c886245:38229 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T19:57:25,305 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 65a3df80a12c37fb9fbab62e0bca8061 2024-11-24T19:57:25,306 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 566da3ef551e53b7c5fe23ca2c3e4c49 2024-11-24T19:57:25,306 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 8b7380c45a53f728d75b96169ee5e67d 2024-11-24T19:57:25,306 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 8b52240251203d3232be1c4491b831e4 2024-11-24T19:57:25,306 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 184906bf8a6e152a8c22472839b543e1 2024-11-24T19:57:25,306 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 7f8a6d58dc9d29ed781574e1b58a0094 2024-11-24T19:57:25,306 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 024525ffa8c3c04085ca40ac2c8fc04e 2024-11-24T19:57:25,306 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 3ca11ada5ddb9634563081985fee5db9 2024-11-24T19:57:25,306 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:25,306 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 65a3df80a12c37fb9fbab62e0bca8061, disabling compactions & flushes 2024-11-24T19:57:25,307 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:57:25,307 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:57:25,307 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:57:25,307 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(959): stopping server 32a03c886245,38229,1732478146827 2024-11-24T19:57:25,307 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. after waiting 0 ms 2024-11-24T19:57:25,307 INFO [RS:0;32a03c886245:38229 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:57:25,307 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:57:25,307 INFO [RS:0;32a03c886245:38229 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;32a03c886245:38229. 2024-11-24T19:57:25,307 DEBUG [RS:0;32a03c886245:38229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T19:57:25,307 DEBUG [RS:0;32a03c886245:38229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:57:25,308 INFO [RS:0;32a03c886245:38229 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T19:57:25,308 INFO [RS:0;32a03c886245:38229 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T19:57:25,308 INFO [RS:0;32a03c886245:38229 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T19:57:25,308 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T19:57:25,309 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-24T19:57:25,309 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T19:57:25,309 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T19:57:25,309 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T19:57:25,310 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T19:57:25,310 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T19:57:25,310 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 1588230740, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 566da3ef551e53b7c5fe23ca2c3e4c49, 65a3df80a12c37fb9fbab62e0bca8061, 7f8a6d58dc9d29ed781574e1b58a0094, 8b52240251203d3232be1c4491b831e4, 8b7380c45a53f728d75b96169ee5e67d, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:25,312 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 65a3df80a12c37fb9fbab62e0bca8061 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:25,312 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-24T19:57:25,329 INFO [regionserver/32a03c886245:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T19:57:25,329 INFO [regionserver/32a03c886245:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T19:57:25,363 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061/.tmp/testFamily/f0c5346f14e348a4922ab6443d43bc5f is 61, key is row-0/testFamily:testQualifier/1732478213849/Put/seqid=0 2024-11-24T19:57:25,364 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-24T19:57:25,370 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/info/32f259a74f814606bccf60f60ef3ae46 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d./info:regioninfo/1732478181435/Put/seqid=0 2024-11-24T19:57:25,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741855_1031 (size=5049) 2024-11-24T19:57:25,375 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061/.tmp/testFamily/f0c5346f14e348a4922ab6443d43bc5f 2024-11-24T19:57:25,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741856_1032 (size=20564) 2024-11-24T19:57:25,380 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/info/32f259a74f814606bccf60f60ef3ae46 2024-11-24T19:57:25,418 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061/.tmp/testFamily/f0c5346f14e348a4922ab6443d43bc5f as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061/testFamily/f0c5346f14e348a4922ab6443d43bc5f 2024-11-24T19:57:25,427 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061/testFamily/f0c5346f14e348a4922ab6443d43bc5f, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:25,437 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65a3df80a12c37fb9fbab62e0bca8061 in 123ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:25,445 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNextAsync/65a3df80a12c37fb9fbab62e0bca8061/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:25,446 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/ns/5e1822da3cbc42ce985264179a714fc6 is 43, key is default/ns:d/1732478179926/Put/seqid=0 2024-11-24T19:57:25,447 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:57:25,447 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 65a3df80a12c37fb9fbab62e0bca8061: Waiting for close lock at 1732478245306Running coprocessor pre-close hooks at 1732478245306Disabling compacts and flushes for region at 1732478245306Disabling writes for close at 1732478245307 (+1 ms)Obtaining lock to block concurrent updates at 1732478245312 (+5 ms)Preparing flush snapshotting stores in 65a3df80a12c37fb9fbab62e0bca8061 at 1732478245312Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478245320 (+8 ms)Flushing stores of testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. at 1732478245321 (+1 ms)Flushing 65a3df80a12c37fb9fbab62e0bca8061/testFamily: creating writer at 1732478245323 (+2 ms)Flushing 65a3df80a12c37fb9fbab62e0bca8061/testFamily: appending metadata at 1732478245358 (+35 ms)Flushing 65a3df80a12c37fb9fbab62e0bca8061/testFamily: closing flushed file at 1732478245360 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52c2667d: reopening flushed file at 1732478245416 (+56 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 65a3df80a12c37fb9fbab62e0bca8061 in 123ms, sequenceid=8, compaction requested=false at 1732478245437 (+21 ms)Writing region close event to WAL at 1732478245439 (+2 ms)Running coprocessor post-close hooks at 1732478245446 (+7 ms)Closed at 1732478245447 (+1 ms) 2024-11-24T19:57:25,448 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732478210800.65a3df80a12c37fb9fbab62e0bca8061. 2024-11-24T19:57:25,448 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 566da3ef551e53b7c5fe23ca2c3e4c49, disabling compactions & flushes 2024-11-24T19:57:25,448 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:57:25,448 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:57:25,448 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. after waiting 0 ms 2024-11-24T19:57:25,448 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:57:25,448 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 566da3ef551e53b7c5fe23ca2c3e4c49 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:25,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741857_1033 (size=5153) 2024-11-24T19:57:25,466 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49/.tmp/testFamily/5b165a8fd1d74a479b9eb66bd8b83072 is 61, key is row-0/testFamily:testQualifier/1732478195537/Put/seqid=0 2024-11-24T19:57:25,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741858_1034 (size=5049) 2024-11-24T19:57:25,510 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 1588230740, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 566da3ef551e53b7c5fe23ca2c3e4c49, 7f8a6d58dc9d29ed781574e1b58a0094, 8b52240251203d3232be1c4491b831e4, 8b7380c45a53f728d75b96169ee5e67d, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:25,710 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 1588230740, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 566da3ef551e53b7c5fe23ca2c3e4c49, 7f8a6d58dc9d29ed781574e1b58a0094, 8b52240251203d3232be1c4491b831e4, 8b7380c45a53f728d75b96169ee5e67d, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:25,769 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-24T19:57:25,873 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/ns/5e1822da3cbc42ce985264179a714fc6 2024-11-24T19:57:25,873 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49/.tmp/testFamily/5b165a8fd1d74a479b9eb66bd8b83072 2024-11-24T19:57:25,888 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49/.tmp/testFamily/5b165a8fd1d74a479b9eb66bd8b83072 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49/testFamily/5b165a8fd1d74a479b9eb66bd8b83072 2024-11-24T19:57:25,896 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49/testFamily/5b165a8fd1d74a479b9eb66bd8b83072, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:25,897 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 566da3ef551e53b7c5fe23ca2c3e4c49 in 449ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:25,903 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNextAsync/566da3ef551e53b7c5fe23ca2c3e4c49/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:25,904 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:57:25,904 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 566da3ef551e53b7c5fe23ca2c3e4c49: Waiting for close lock at 1732478245448Running coprocessor pre-close hooks at 1732478245448Disabling compacts and flushes for region at 1732478245448Disabling writes for close at 1732478245448Obtaining lock to block concurrent updates at 1732478245448Preparing flush snapshotting stores in 566da3ef551e53b7c5fe23ca2c3e4c49 at 1732478245448Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478245449 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. at 1732478245449Flushing 566da3ef551e53b7c5fe23ca2c3e4c49/testFamily: creating writer at 1732478245449Flushing 566da3ef551e53b7c5fe23ca2c3e4c49/testFamily: appending metadata at 1732478245466 (+17 ms)Flushing 566da3ef551e53b7c5fe23ca2c3e4c49/testFamily: closing flushed file at 1732478245466Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dd9be24: reopening flushed file at 1732478245886 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 566da3ef551e53b7c5fe23ca2c3e4c49 in 449ms, sequenceid=8, compaction requested=false at 1732478245897 (+11 ms)Writing region close event to WAL at 1732478245899 (+2 ms)Running coprocessor post-close hooks at 1732478245904 (+5 ms)Closed at 1732478245904 2024-11-24T19:57:25,904 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732478192475.566da3ef551e53b7c5fe23ca2c3e4c49. 2024-11-24T19:57:25,904 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/table/d90d8d2bf75d4cfea539bb6748bc3f48 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732478181450/Put/seqid=0 2024-11-24T19:57:25,904 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8b7380c45a53f728d75b96169ee5e67d, disabling compactions & flushes 2024-11-24T19:57:25,904 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:57:25,904 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:57:25,904 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. after waiting 0 ms 2024-11-24T19:57:25,904 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:57:25,904 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8b7380c45a53f728d75b96169ee5e67d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:25,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741859_1035 (size=6699) 2024-11-24T19:57:25,911 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 1588230740, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, 8b52240251203d3232be1c4491b831e4, 8b7380c45a53f728d75b96169ee5e67d, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:25,921 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d/.tmp/testFamily/48b0182468e34ee2ba54c8478cb0249b is 61, key is row-0/testFamily:testQualifier/1732478183272/Put/seqid=0 2024-11-24T19:57:25,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741860_1036 (size=5049) 2024-11-24T19:57:26,111 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 1588230740, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, 8b52240251203d3232be1c4491b831e4, 8b7380c45a53f728d75b96169ee5e67d, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:26,311 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-24T19:57:26,312 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/table/d90d8d2bf75d4cfea539bb6748bc3f48 2024-11-24T19:57:26,312 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1325): Online Regions={8b7380c45a53f728d75b96169ee5e67d=testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d., 8b52240251203d3232be1c4491b831e4=testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., 184906bf8a6e152a8c22472839b543e1=testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1., 1588230740=hbase:meta,,1.1588230740, 7f8a6d58dc9d29ed781574e1b58a0094=testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094., 024525ffa8c3c04085ca40ac2c8fc04e=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., 3ca11ada5ddb9634563081985fee5db9=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., eede8119663a58fe5707f1b4ce9249e4=hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4., 52407b6249c71db9494e29d4d16771fd=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.} 2024-11-24T19:57:26,312 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 1588230740, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, 8b52240251203d3232be1c4491b831e4, 8b7380c45a53f728d75b96169ee5e67d, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:26,327 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d/.tmp/testFamily/48b0182468e34ee2ba54c8478cb0249b 2024-11-24T19:57:26,328 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/info/32f259a74f814606bccf60f60ef3ae46 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/info/32f259a74f814606bccf60f60ef3ae46 2024-11-24T19:57:26,335 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d/.tmp/testFamily/48b0182468e34ee2ba54c8478cb0249b as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d/testFamily/48b0182468e34ee2ba54c8478cb0249b 2024-11-24T19:57:26,337 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/info/32f259a74f814606bccf60f60ef3ae46, entries=100, sequenceid=56, filesize=20.1 K 2024-11-24T19:57:26,338 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/ns/5e1822da3cbc42ce985264179a714fc6 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/ns/5e1822da3cbc42ce985264179a714fc6 2024-11-24T19:57:26,343 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d/testFamily/48b0182468e34ee2ba54c8478cb0249b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:26,344 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8b7380c45a53f728d75b96169ee5e67d in 440ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:26,348 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/ns/5e1822da3cbc42ce985264179a714fc6, entries=2, sequenceid=56, filesize=5.0 K 2024-11-24T19:57:26,349 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/8b7380c45a53f728d75b96169ee5e67d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:26,350 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:57:26,350 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/.tmp/table/d90d8d2bf75d4cfea539bb6748bc3f48 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/table/d90d8d2bf75d4cfea539bb6748bc3f48 2024-11-24T19:57:26,350 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8b7380c45a53f728d75b96169ee5e67d: Waiting for close lock at 1732478245904Running coprocessor pre-close hooks at 1732478245904Disabling compacts and flushes for region at 1732478245904Disabling writes for close at 1732478245904Obtaining lock to block concurrent updates at 1732478245904Preparing flush snapshotting stores in 8b7380c45a53f728d75b96169ee5e67d at 1732478245904Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478245905 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. at 1732478245905Flushing 8b7380c45a53f728d75b96169ee5e67d/testFamily: creating writer at 1732478245905Flushing 8b7380c45a53f728d75b96169ee5e67d/testFamily: appending metadata at 1732478245921 (+16 ms)Flushing 8b7380c45a53f728d75b96169ee5e67d/testFamily: closing flushed file at 1732478245921Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f81eeb0: reopening flushed file at 1732478246334 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8b7380c45a53f728d75b96169ee5e67d in 440ms, sequenceid=8, compaction requested=false at 1732478246344 (+10 ms)Writing region close event to WAL at 1732478246345 (+1 ms)Running coprocessor post-close hooks at 1732478246349 (+4 ms)Closed at 1732478246350 (+1 ms) 2024-11-24T19:57:26,350 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732478180203.8b7380c45a53f728d75b96169ee5e67d. 2024-11-24T19:57:26,350 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8b52240251203d3232be1c4491b831e4, disabling compactions & flushes 2024-11-24T19:57:26,350 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:57:26,350 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:57:26,350 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. after waiting 0 ms 2024-11-24T19:57:26,350 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:57:26,350 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8b52240251203d3232be1c4491b831e4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:26,358 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/table/d90d8d2bf75d4cfea539bb6748bc3f48, entries=20, sequenceid=56, filesize=6.5 K 2024-11-24T19:57:26,359 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1049ms, sequenceid=56, compaction requested=false 2024-11-24T19:57:26,367 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-24T19:57:26,368 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T19:57:26,368 INFO [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T19:57:26,368 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732478245309Running coprocessor pre-close hooks at 1732478245309Disabling compacts and flushes for region at 1732478245309Disabling writes for close at 1732478245310 (+1 ms)Obtaining lock to block concurrent updates at 1732478245312 (+2 ms)Preparing flush snapshotting stores in 1588230740 at 1732478245312Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732478245320 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732478245321 (+1 ms)Flushing 1588230740/info: creating writer at 1732478245323 (+2 ms)Flushing 1588230740/info: appending metadata at 1732478245364 (+41 ms)Flushing 1588230740/info: closing flushed file at 1732478245364Flushing 1588230740/ns: creating writer at 1732478245416 (+52 ms)Flushing 1588230740/ns: appending metadata at 1732478245445 (+29 ms)Flushing 1588230740/ns: closing flushed file at 1732478245445Flushing 1588230740/table: creating writer at 1732478245887 (+442 ms)Flushing 1588230740/table: appending metadata at 1732478245903 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732478245903Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c29f7e6: reopening flushed file at 1732478246327 (+424 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@626f29d3: reopening flushed file at 1732478246337 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba11d44: reopening flushed file at 1732478246348 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1049ms, sequenceid=56, compaction requested=false at 1732478246359 (+11 ms)Writing region close event to WAL at 1732478246363 (+4 ms)Running coprocessor post-close hooks at 1732478246368 (+5 ms)Closed at 1732478246368 2024-11-24T19:57:26,368 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4/.tmp/testFamily/6acf05c9efbe44a792249cca16907757 is 61, key is row-0/testFamily:testQualifier/1732478207748/Put/seqid=0 2024-11-24T19:57:26,368 DEBUG [RS_CLOSE_META-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T19:57:26,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741861_1037 (size=5049) 2024-11-24T19:57:26,451 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-24T19:57:26,452 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-24T19:57:26,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T19:57:26,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-24T19:57:26,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T19:57:26,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-24T19:57:26,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-24T19:57:26,513 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, 8b52240251203d3232be1c4491b831e4, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:26,713 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, 8b52240251203d3232be1c4491b831e4, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:26,775 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4/.tmp/testFamily/6acf05c9efbe44a792249cca16907757 2024-11-24T19:57:26,787 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4/.tmp/testFamily/6acf05c9efbe44a792249cca16907757 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4/testFamily/6acf05c9efbe44a792249cca16907757 2024-11-24T19:57:26,793 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4/testFamily/6acf05c9efbe44a792249cca16907757, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:26,794 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8b52240251203d3232be1c4491b831e4 in 444ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:26,799 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScanner/8b52240251203d3232be1c4491b831e4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:26,800 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:57:26,800 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8b52240251203d3232be1c4491b831e4: Waiting for close lock at 1732478246350Running coprocessor pre-close hooks at 1732478246350Disabling compacts and flushes for region at 1732478246350Disabling writes for close at 1732478246350Obtaining lock to block concurrent updates at 1732478246350Preparing flush snapshotting stores in 8b52240251203d3232be1c4491b831e4 at 1732478246350Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478246351 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. at 1732478246351Flushing 8b52240251203d3232be1c4491b831e4/testFamily: creating writer at 1732478246351Flushing 8b52240251203d3232be1c4491b831e4/testFamily: appending metadata at 1732478246368 (+17 ms)Flushing 8b52240251203d3232be1c4491b831e4/testFamily: closing flushed file at 1732478246368Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7956657a: reopening flushed file at 1732478246785 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8b52240251203d3232be1c4491b831e4 in 444ms, sequenceid=8, compaction requested=false at 1732478246794 (+9 ms)Writing region close event to WAL at 1732478246795 (+1 ms)Running coprocessor post-close hooks at 1732478246800 (+5 ms)Closed at 1732478246800 2024-11-24T19:57:26,800 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732478204701.8b52240251203d3232be1c4491b831e4. 2024-11-24T19:57:26,800 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 184906bf8a6e152a8c22472839b543e1, disabling compactions & flushes 2024-11-24T19:57:26,800 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:57:26,800 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:57:26,800 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. after waiting 0 ms 2024-11-24T19:57:26,800 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:57:26,800 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 184906bf8a6e152a8c22472839b543e1 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:26,815 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1/.tmp/testFamily/a759e7a2442f48638df42a4feec77985 is 61, key is row-0/testFamily:testQualifier/1732478219955/Put/seqid=0 2024-11-24T19:57:26,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741862_1038 (size=5049) 2024-11-24T19:57:26,913 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:27,097 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=32a03c886245:45559 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-24T19:57:27,114 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 184906bf8a6e152a8c22472839b543e1, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:27,221 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1/.tmp/testFamily/a759e7a2442f48638df42a4feec77985 2024-11-24T19:57:27,233 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1/.tmp/testFamily/a759e7a2442f48638df42a4feec77985 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1/testFamily/a759e7a2442f48638df42a4feec77985 2024-11-24T19:57:27,242 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1/testFamily/a759e7a2442f48638df42a4feec77985, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:27,243 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 184906bf8a6e152a8c22472839b543e1 in 443ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:27,249 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnNext/184906bf8a6e152a8c22472839b543e1/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:27,250 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:57:27,250 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 184906bf8a6e152a8c22472839b543e1: Waiting for close lock at 1732478246800Running coprocessor pre-close hooks at 1732478246800Disabling compacts and flushes for region at 1732478246800Disabling writes for close at 1732478246800Obtaining lock to block concurrent updates at 1732478246800Preparing flush snapshotting stores in 184906bf8a6e152a8c22472839b543e1 at 1732478246800Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478246801 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. at 1732478246801Flushing 184906bf8a6e152a8c22472839b543e1/testFamily: creating writer at 1732478246801Flushing 184906bf8a6e152a8c22472839b543e1/testFamily: appending metadata at 1732478246814 (+13 ms)Flushing 184906bf8a6e152a8c22472839b543e1/testFamily: closing flushed file at 1732478246814Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24e9516b: reopening flushed file at 1732478247232 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 184906bf8a6e152a8c22472839b543e1 in 443ms, sequenceid=8, compaction requested=false at 1732478247243 (+11 ms)Writing region close event to WAL at 1732478247244 (+1 ms)Running coprocessor post-close hooks at 1732478247250 (+6 ms)Closed at 1732478247250 2024-11-24T19:57:27,250 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732478216890.184906bf8a6e152a8c22472839b543e1. 2024-11-24T19:57:27,251 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7f8a6d58dc9d29ed781574e1b58a0094, disabling compactions & flushes 2024-11-24T19:57:27,251 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:57:27,251 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:57:27,251 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. after waiting 0 ms 2024-11-24T19:57:27,251 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:57:27,251 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 7f8a6d58dc9d29ed781574e1b58a0094 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:27,265 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094/.tmp/testFamily/98754d8f4007428ea196f22d5833f637 is 61, key is row-0/testFamily:testQualifier/1732478204646/Put/seqid=0 2024-11-24T19:57:27,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741863_1039 (size=5049) 2024-11-24T19:57:27,314 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-24T19:57:27,314 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1325): Online Regions={7f8a6d58dc9d29ed781574e1b58a0094=testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094., 024525ffa8c3c04085ca40ac2c8fc04e=testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., 3ca11ada5ddb9634563081985fee5db9=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., eede8119663a58fe5707f1b4ce9249e4=hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4., 52407b6249c71db9494e29d4d16771fd=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.} 2024-11-24T19:57:27,314 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:27,452 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T19:57:27,514 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, 7f8a6d58dc9d29ed781574e1b58a0094, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:27,671 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094/.tmp/testFamily/98754d8f4007428ea196f22d5833f637 2024-11-24T19:57:27,679 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094/.tmp/testFamily/98754d8f4007428ea196f22d5833f637 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094/testFamily/98754d8f4007428ea196f22d5833f637 2024-11-24T19:57:27,687 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094/testFamily/98754d8f4007428ea196f22d5833f637, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:27,688 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7f8a6d58dc9d29ed781574e1b58a0094 in 437ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:27,693 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testRetryOutOfOrderScannerNextException/7f8a6d58dc9d29ed781574e1b58a0094/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:27,694 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:57:27,694 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7f8a6d58dc9d29ed781574e1b58a0094: Waiting for close lock at 1732478247251Running coprocessor pre-close hooks at 1732478247251Disabling compacts and flushes for region at 1732478247251Disabling writes for close at 1732478247251Obtaining lock to block concurrent updates at 1732478247251Preparing flush snapshotting stores in 7f8a6d58dc9d29ed781574e1b58a0094 at 1732478247251Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478247251Flushing stores of testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. at 1732478247252 (+1 ms)Flushing 7f8a6d58dc9d29ed781574e1b58a0094/testFamily: creating writer at 1732478247252Flushing 7f8a6d58dc9d29ed781574e1b58a0094/testFamily: appending metadata at 1732478247265 (+13 ms)Flushing 7f8a6d58dc9d29ed781574e1b58a0094/testFamily: closing flushed file at 1732478247265Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6039f3aa: reopening flushed file at 1732478247678 (+413 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 7f8a6d58dc9d29ed781574e1b58a0094 in 437ms, sequenceid=8, compaction requested=false at 1732478247689 (+11 ms)Writing region close event to WAL at 1732478247689Running coprocessor post-close hooks at 1732478247694 (+5 ms)Closed at 1732478247694 2024-11-24T19:57:27,694 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732478201589.7f8a6d58dc9d29ed781574e1b58a0094. 2024-11-24T19:57:27,694 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 024525ffa8c3c04085ca40ac2c8fc04e, disabling compactions & flushes 2024-11-24T19:57:27,694 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:27,694 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:27,694 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. after waiting 0 ms 2024-11-24T19:57:27,695 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:27,695 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 024525ffa8c3c04085ca40ac2c8fc04e 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:27,710 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e/.tmp/testFamily/62009fa4a9e948f1a34beb742d96a7d4 is 61, key is row-0/testFamily:testQualifier/1732478242217/Put/seqid=0 2024-11-24T19:57:27,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741864_1040 (size=5049) 2024-11-24T19:57:27,715 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:27,915 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:28,115 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 024525ffa8c3c04085ca40ac2c8fc04e, 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:28,117 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e/.tmp/testFamily/62009fa4a9e948f1a34beb742d96a7d4 2024-11-24T19:57:28,133 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e/.tmp/testFamily/62009fa4a9e948f1a34beb742d96a7d4 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e/testFamily/62009fa4a9e948f1a34beb742d96a7d4 2024-11-24T19:57:28,140 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e/testFamily/62009fa4a9e948f1a34beb742d96a7d4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:28,141 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 024525ffa8c3c04085ca40ac2c8fc04e in 446ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:28,146 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/default/testNormalScanTimeoutOnOpenScannerAsync/024525ffa8c3c04085ca40ac2c8fc04e/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:28,146 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:28,146 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 024525ffa8c3c04085ca40ac2c8fc04e: Waiting for close lock at 1732478247694Running coprocessor pre-close hooks at 1732478247694Disabling compacts and flushes for region at 1732478247694Disabling writes for close at 1732478247695 (+1 ms)Obtaining lock to block concurrent updates at 1732478247695Preparing flush snapshotting stores in 024525ffa8c3c04085ca40ac2c8fc04e at 1732478247695Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478247695Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. at 1732478247696 (+1 ms)Flushing 024525ffa8c3c04085ca40ac2c8fc04e/testFamily: creating writer at 1732478247696Flushing 024525ffa8c3c04085ca40ac2c8fc04e/testFamily: appending metadata at 1732478247709 (+13 ms)Flushing 024525ffa8c3c04085ca40ac2c8fc04e/testFamily: closing flushed file at 1732478247710 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32156e3f: reopening flushed file at 1732478248132 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 024525ffa8c3c04085ca40ac2c8fc04e in 446ms, sequenceid=8, compaction requested=false at 1732478248141 (+9 ms)Writing region close event to WAL at 1732478248142 (+1 ms)Running coprocessor post-close hooks at 1732478248146 (+4 ms)Closed at 1732478248146 2024-11-24T19:57:28,147 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732478241186.024525ffa8c3c04085ca40ac2c8fc04e. 2024-11-24T19:57:28,147 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3ca11ada5ddb9634563081985fee5db9, disabling compactions & flushes 2024-11-24T19:57:28,147 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:28,147 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:28,147 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. after waiting 0 ms 2024-11-24T19:57:28,147 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:28,147 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3ca11ada5ddb9634563081985fee5db9 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:28,164 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9/.tmp/testFamily/8ecd6f01689b4837b45731bc65fcebde is 61, key is row-0/testFamily:testQualifier/1732478226068/Put/seqid=0 2024-11-24T19:57:28,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741865_1041 (size=5049) 2024-11-24T19:57:28,316 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-24T19:57:28,316 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1325): Online Regions={3ca11ada5ddb9634563081985fee5db9=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., eede8119663a58fe5707f1b4ce9249e4=hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4., 52407b6249c71db9494e29d4d16771fd=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.} 2024-11-24T19:57:28,316 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:28,517 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 3ca11ada5ddb9634563081985fee5db9, 52407b6249c71db9494e29d4d16771fd, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:28,570 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9/.tmp/testFamily/8ecd6f01689b4837b45731bc65fcebde 2024-11-24T19:57:28,583 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9/.tmp/testFamily/8ecd6f01689b4837b45731bc65fcebde as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9/testFamily/8ecd6f01689b4837b45731bc65fcebde 2024-11-24T19:57:28,589 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9/testFamily/8ecd6f01689b4837b45731bc65fcebde, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:28,590 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3ca11ada5ddb9634563081985fee5db9 in 443ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:28,594 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/3ca11ada5ddb9634563081985fee5db9/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:28,595 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:28,595 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3ca11ada5ddb9634563081985fee5db9: Waiting for close lock at 1732478248147Running coprocessor pre-close hooks at 1732478248147Disabling compacts and flushes for region at 1732478248147Disabling writes for close at 1732478248147Obtaining lock to block concurrent updates at 1732478248147Preparing flush snapshotting stores in 3ca11ada5ddb9634563081985fee5db9 at 1732478248147Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478248147Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. at 1732478248148 (+1 ms)Flushing 3ca11ada5ddb9634563081985fee5db9/testFamily: creating writer at 1732478248148Flushing 3ca11ada5ddb9634563081985fee5db9/testFamily: appending metadata at 1732478248163 (+15 ms)Flushing 3ca11ada5ddb9634563081985fee5db9/testFamily: closing flushed file at 1732478248163Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b60b6d9: reopening flushed file at 1732478248582 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3ca11ada5ddb9634563081985fee5db9 in 443ms, sequenceid=8, compaction requested=false at 1732478248590 (+8 ms)Writing region close event to WAL at 1732478248591 (+1 ms)Running coprocessor post-close hooks at 1732478248595 (+4 ms)Closed at 1732478248595 2024-11-24T19:57:28,595 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732478223010.3ca11ada5ddb9634563081985fee5db9. 2024-11-24T19:57:28,595 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eede8119663a58fe5707f1b4ce9249e4, disabling compactions & flushes 2024-11-24T19:57:28,595 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:28,595 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:28,596 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. after waiting 0 ms 2024-11-24T19:57:28,596 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:28,596 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing eede8119663a58fe5707f1b4ce9249e4 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:28,612 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4/.tmp/testFamily/1d736ff93a0d4b5eb25aa61faa7073cb is 61, key is row-0/testFamily:testQualifier/1732478235142/Put/seqid=0 2024-11-24T19:57:28,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741866_1042 (size=5049) 2024-11-24T19:57:28,717 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 52407b6249c71db9494e29d4d16771fd, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:28,917 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 52407b6249c71db9494e29d4d16771fd, eede8119663a58fe5707f1b4ce9249e4 2024-11-24T19:57:29,019 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4/.tmp/testFamily/1d736ff93a0d4b5eb25aa61faa7073cb 2024-11-24T19:57:29,032 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4/.tmp/testFamily/1d736ff93a0d4b5eb25aa61faa7073cb as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4/testFamily/1d736ff93a0d4b5eb25aa61faa7073cb 2024-11-24T19:57:29,039 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4/testFamily/1d736ff93a0d4b5eb25aa61faa7073cb, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:29,040 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eede8119663a58fe5707f1b4ce9249e4 in 444ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:29,044 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnNext/eede8119663a58fe5707f1b4ce9249e4/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:29,045 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:29,045 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eede8119663a58fe5707f1b4ce9249e4: Waiting for close lock at 1732478248595Running coprocessor pre-close hooks at 1732478248595Disabling compacts and flushes for region at 1732478248595Disabling writes for close at 1732478248596 (+1 ms)Obtaining lock to block concurrent updates at 1732478248596Preparing flush snapshotting stores in eede8119663a58fe5707f1b4ce9249e4 at 1732478248596Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478248596Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. at 1732478248596Flushing eede8119663a58fe5707f1b4ce9249e4/testFamily: creating writer at 1732478248597 (+1 ms)Flushing eede8119663a58fe5707f1b4ce9249e4/testFamily: appending metadata at 1732478248612 (+15 ms)Flushing eede8119663a58fe5707f1b4ce9249e4/testFamily: closing flushed file at 1732478248612Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@732056c: reopening flushed file at 1732478249031 (+419 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for eede8119663a58fe5707f1b4ce9249e4 in 444ms, sequenceid=8, compaction requested=false at 1732478249040 (+9 ms)Writing region close event to WAL at 1732478249040Running coprocessor post-close hooks at 1732478249045 (+5 ms)Closed at 1732478249045 2024-11-24T19:57:29,045 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732478232109.eede8119663a58fe5707f1b4ce9249e4. 2024-11-24T19:57:29,045 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 52407b6249c71db9494e29d4d16771fd, disabling compactions & flushes 2024-11-24T19:57:29,045 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:57:29,045 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:57:29,045 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. after waiting 0 ms 2024-11-24T19:57:29,045 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:57:29,045 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 52407b6249c71db9494e29d4d16771fd 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-24T19:57:29,060 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd/.tmp/testFamily/5fdad79cf1d04450805ae962491fb896 is 61, key is row-0/testFamily:testQualifier/1732478186419/Put/seqid=0 2024-11-24T19:57:29,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741867_1043 (size=5049) 2024-11-24T19:57:29,118 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:57:29,318 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T19:57:29,318 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1325): Online Regions={52407b6249c71db9494e29d4d16771fd=hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd.} 2024-11-24T19:57:29,318 DEBUG [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1351): Waiting on 52407b6249c71db9494e29d4d16771fd 2024-11-24T19:57:29,467 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd/.tmp/testFamily/5fdad79cf1d04450805ae962491fb896 2024-11-24T19:57:29,483 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd/.tmp/testFamily/5fdad79cf1d04450805ae962491fb896 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd/testFamily/5fdad79cf1d04450805ae962491fb896 2024-11-24T19:57:29,493 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd/testFamily/5fdad79cf1d04450805ae962491fb896, entries=4, sequenceid=8, filesize=4.9 K 2024-11-24T19:57:29,494 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 52407b6249c71db9494e29d4d16771fd in 449ms, sequenceid=8, compaction requested=false 2024-11-24T19:57:29,500 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/data/hbase/testMetaScanTimeoutOnOpenScanner/52407b6249c71db9494e29d4d16771fd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-24T19:57:29,501 INFO [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:57:29,501 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 52407b6249c71db9494e29d4d16771fd: Waiting for close lock at 1732478249045Running coprocessor pre-close hooks at 1732478249045Disabling compacts and flushes for region at 1732478249045Disabling writes for close at 1732478249045Obtaining lock to block concurrent updates at 1732478249045Preparing flush snapshotting stores in 52407b6249c71db9494e29d4d16771fd at 1732478249045Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732478249045Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. at 1732478249046 (+1 ms)Flushing 52407b6249c71db9494e29d4d16771fd/testFamily: creating writer at 1732478249046Flushing 52407b6249c71db9494e29d4d16771fd/testFamily: appending metadata at 1732478249059 (+13 ms)Flushing 52407b6249c71db9494e29d4d16771fd/testFamily: closing flushed file at 1732478249060 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44deaa3c: reopening flushed file at 1732478249482 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 52407b6249c71db9494e29d4d16771fd in 449ms, sequenceid=8, compaction requested=false at 1732478249494 (+12 ms)Writing region close event to WAL at 1732478249495 (+1 ms)Running coprocessor post-close hooks at 1732478249501 (+6 ms)Closed at 1732478249501 2024-11-24T19:57:29,501 DEBUG [RS_CLOSE_REGION-regionserver/32a03c886245:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732478183374.52407b6249c71db9494e29d4d16771fd. 2024-11-24T19:57:29,519 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(976): stopping server 32a03c886245,38229,1732478146827; all regions closed. 2024-11-24T19:57:29,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741834_1010 (size=19479) 2024-11-24T19:57:29,534 DEBUG [RS:0;32a03c886245:38229 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/oldWALs 2024-11-24T19:57:29,534 INFO [RS:0;32a03c886245:38229 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C38229%2C1732478146827.meta:.meta(num 1732478179734) 2024-11-24T19:57:29,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741833_1009 (size=20031) 2024-11-24T19:57:29,537 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/WALs/32a03c886245,38229,1732478146827/32a03c886245%2C38229%2C1732478146827.1732478179387 not finished, retry = 0 2024-11-24T19:57:29,642 DEBUG [RS:0;32a03c886245:38229 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/oldWALs 2024-11-24T19:57:29,642 INFO [RS:0;32a03c886245:38229 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 32a03c886245%2C38229%2C1732478146827:(num 1732478179387) 2024-11-24T19:57:29,643 DEBUG [RS:0;32a03c886245:38229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T19:57:29,643 INFO [RS:0;32a03c886245:38229 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:57:29,643 INFO [RS:0;32a03c886245:38229 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:57:29,644 INFO [RS:0;32a03c886245:38229 {}] hbase.ChoreService(370): Chore service for: regionserver/32a03c886245:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T19:57:29,644 INFO [RS:0;32a03c886245:38229 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:57:29,644 INFO [regionserver/32a03c886245:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:57:29,646 INFO [RS:0;32a03c886245:38229 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38229 2024-11-24T19:57:29,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T19:57:29,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/32a03c886245,38229,1732478146827 2024-11-24T19:57:29,708 INFO [RS:0;32a03c886245:38229 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:57:29,717 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [32a03c886245,38229,1732478146827] 2024-11-24T19:57:29,724 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/32a03c886245,38229,1732478146827 already deleted, retry=false 2024-11-24T19:57:29,725 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 32a03c886245,38229,1732478146827 expired; onlineServers=0 2024-11-24T19:57:29,725 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '32a03c886245,45559,1732478146124' ***** 2024-11-24T19:57:29,725 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T19:57:29,725 INFO [M:0;32a03c886245:45559 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T19:57:29,725 INFO [M:0;32a03c886245:45559 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T19:57:29,726 DEBUG [M:0;32a03c886245:45559 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T19:57:29,726 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T19:57:29,726 DEBUG [M:0;32a03c886245:45559 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T19:57:29,726 DEBUG [master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478149153 {}] cleaner.HFileCleaner(306): Exit Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.small.0-1732478149153,5,FailOnTimeoutGroup] 2024-11-24T19:57:29,726 DEBUG [master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478149147 {}] cleaner.HFileCleaner(306): Exit Thread[master/32a03c886245:0:becomeActiveMaster-HFileCleaner.large.0-1732478149147,5,FailOnTimeoutGroup] 2024-11-24T19:57:29,727 INFO [M:0;32a03c886245:45559 {}] hbase.ChoreService(370): Chore service for: master/32a03c886245:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T19:57:29,727 INFO [M:0;32a03c886245:45559 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T19:57:29,727 DEBUG [M:0;32a03c886245:45559 {}] master.HMaster(1795): Stopping service threads 2024-11-24T19:57:29,727 INFO [M:0;32a03c886245:45559 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T19:57:29,728 INFO [M:0;32a03c886245:45559 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T19:57:29,729 INFO [M:0;32a03c886245:45559 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T19:57:29,729 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T19:57:29,738 INFO [regionserver/32a03c886245:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T19:57:29,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T19:57:29,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T19:57:29,739 DEBUG [M:0;32a03c886245:45559 {}] zookeeper.ZKUtil(347): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T19:57:29,739 WARN [M:0;32a03c886245:45559 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T19:57:29,740 INFO [M:0;32a03c886245:45559 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/.lastflushedseqids 2024-11-24T19:57:29,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741868_1044 (size=649) 2024-11-24T19:57:29,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:57:29,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38229-0x1016fb425e70001, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:57:29,818 INFO [RS:0;32a03c886245:38229 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:57:29,819 INFO [RS:0;32a03c886245:38229 {}] regionserver.HRegionServer(1031): Exiting; stopping=32a03c886245,38229,1732478146827; zookeeper connection closed. 2024-11-24T19:57:29,819 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@23944a50 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@23944a50 2024-11-24T19:57:29,820 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T19:57:30,156 INFO [M:0;32a03c886245:45559 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T19:57:30,156 INFO [M:0;32a03c886245:45559 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T19:57:30,157 DEBUG [M:0;32a03c886245:45559 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T19:57:30,157 INFO [M:0;32a03c886245:45559 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:57:30,157 DEBUG [M:0;32a03c886245:45559 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:57:30,158 DEBUG [M:0;32a03c886245:45559 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T19:57:30,158 DEBUG [M:0;32a03c886245:45559 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:57:30,159 INFO [M:0;32a03c886245:45559 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.11 KB heapSize=191 KB 2024-11-24T19:57:30,179 DEBUG [M:0;32a03c886245:45559 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b73bfecfe9fc46bc87528ae2212556d7 is 82, key is hbase:meta,,1/info:regioninfo/1732478179815/Put/seqid=0 2024-11-24T19:57:30,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741869_1045 (size=5672) 2024-11-24T19:57:30,586 INFO [M:0;32a03c886245:45559 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b73bfecfe9fc46bc87528ae2212556d7 2024-11-24T19:57:30,620 DEBUG [M:0;32a03c886245:45559 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4c5ea81e857a4cb19b336163d6e597fa is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732478181458/Put/seqid=0 2024-11-24T19:57:30,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741870_1046 (size=14080) 2024-11-24T19:57:31,027 INFO [M:0;32a03c886245:45559 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.55 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4c5ea81e857a4cb19b336163d6e597fa 2024-11-24T19:57:31,038 INFO [M:0;32a03c886245:45559 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4c5ea81e857a4cb19b336163d6e597fa 2024-11-24T19:57:31,054 DEBUG [M:0;32a03c886245:45559 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b948c70df4546ff9d1a2311548d5b75 is 69, key is 32a03c886245,38229,1732478146827/rs:state/1732478149194/Put/seqid=0 2024-11-24T19:57:31,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741871_1047 (size=5156) 2024-11-24T19:57:31,460 INFO [M:0;32a03c886245:45559 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b948c70df4546ff9d1a2311548d5b75 2024-11-24T19:57:31,502 DEBUG [M:0;32a03c886245:45559 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b73bfecfe9fc46bc87528ae2212556d7 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b73bfecfe9fc46bc87528ae2212556d7 2024-11-24T19:57:31,511 INFO [M:0;32a03c886245:45559 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b73bfecfe9fc46bc87528ae2212556d7, entries=8, sequenceid=317, filesize=5.5 K 2024-11-24T19:57:31,513 DEBUG [M:0;32a03c886245:45559 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4c5ea81e857a4cb19b336163d6e597fa as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4c5ea81e857a4cb19b336163d6e597fa 2024-11-24T19:57:31,520 INFO [M:0;32a03c886245:45559 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4c5ea81e857a4cb19b336163d6e597fa 2024-11-24T19:57:31,520 INFO [M:0;32a03c886245:45559 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4c5ea81e857a4cb19b336163d6e597fa, entries=33, sequenceid=317, filesize=13.8 K 2024-11-24T19:57:31,521 DEBUG [M:0;32a03c886245:45559 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6b948c70df4546ff9d1a2311548d5b75 as hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6b948c70df4546ff9d1a2311548d5b75 2024-11-24T19:57:31,527 INFO [M:0;32a03c886245:45559 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35739/user/jenkins/test-data/e34da397-508b-896e-c8a8-036fc028829f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6b948c70df4546ff9d1a2311548d5b75, entries=1, sequenceid=317, filesize=5.0 K 2024-11-24T19:57:31,528 INFO [M:0;32a03c886245:45559 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.11 KB/167023, heapSize ~190.70 KB/195280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1370ms, sequenceid=317, compaction requested=false 2024-11-24T19:57:31,531 INFO [M:0;32a03c886245:45559 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T19:57:31,531 DEBUG [M:0;32a03c886245:45559 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732478250157Disabling compacts and flushes for region at 1732478250157Disabling writes for close at 1732478250158 (+1 ms)Obtaining lock to block concurrent updates at 1732478250159 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732478250159Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=167023, getHeapSize=195520, getOffHeapSize=0, getCellsCount=364 at 1732478250160 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732478250162 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732478250162Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732478250179 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732478250179Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732478250601 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732478250619 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732478250619Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732478251038 (+419 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732478251053 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732478251053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75b61e5f: reopening flushed file at 1732478251500 (+447 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25dce614: reopening flushed file at 1732478251512 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17f1376e: reopening flushed file at 1732478251520 (+8 ms)Finished flush of dataSize ~163.11 KB/167023, heapSize ~190.70 KB/195280, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1370ms, sequenceid=317, compaction requested=false at 1732478251529 (+9 ms)Writing region close event to WAL at 1732478251531 (+2 ms)Closed at 1732478251531 2024-11-24T19:57:31,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39895 is added to blk_1073741830_1006 (size=188235) 2024-11-24T19:57:31,535 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T19:57:31,535 INFO [M:0;32a03c886245:45559 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T19:57:31,535 INFO [M:0;32a03c886245:45559 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45559 2024-11-24T19:57:31,535 INFO [M:0;32a03c886245:45559 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T19:57:31,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:57:31,649 INFO [M:0;32a03c886245:45559 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T19:57:31,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45559-0x1016fb425e70000, quorum=127.0.0.1:53455, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T19:57:31,653 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@429b6cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T19:57:31,655 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T19:57:31,656 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T19:57:31,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T19:57:31,656 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/hadoop.log.dir/,STOPPED} 2024-11-24T19:57:31,658 WARN [BP-2147221055-172.17.0.2-1732478142511 heartbeating to localhost/127.0.0.1:35739 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T19:57:31,658 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T19:57:31,658 WARN [BP-2147221055-172.17.0.2-1732478142511 heartbeating to localhost/127.0.0.1:35739 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2147221055-172.17.0.2-1732478142511 (Datanode Uuid fc68c959-dc9b-48fd-b5bc-f305ebf6a986) service to localhost/127.0.0.1:35739 2024-11-24T19:57:31,658 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T19:57:31,659 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/cluster_ea2b57c6-c725-4370-9708-e0f92e318f26/data/data1/current/BP-2147221055-172.17.0.2-1732478142511 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T19:57:31,659 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/cluster_ea2b57c6-c725-4370-9708-e0f92e318f26/data/data2/current/BP-2147221055-172.17.0.2-1732478142511 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T19:57:31,660 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T19:57:31,667 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T19:57:31,668 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T19:57:31,668 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T19:57:31,668 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T19:57:31,668 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/83dc67b5-cb38-95b5-3703-64b6c1bb54d5/hadoop.log.dir/,STOPPED} 2024-11-24T19:57:31,676 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T19:57:31,707 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down